From 5d9a7b4b8647a72b32da15890e1a321df7ec3701 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 11 Apr 2016 20:21:40 +0100 Subject: [PATCH 01/49] Changed local/global functions to refer to self for inheritance --- RNN.lua | 62 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/RNN.lua b/RNN.lua index 0b7ac09..b48bc48 100644 --- a/RNN.lua +++ b/RNN.lua @@ -31,6 +31,14 @@ function RNN:__init(inputSize, hiddenSize, numLayers) self:reset() end +function RNN:resizeOutput(tensor) + return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize) +end + +function RNN:resizeHidden(tensor) + return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize) +end + function RNN:reset(stdv) stdv = stdv or 1.0 / math.sqrt(self.hiddenSize) @@ -50,7 +58,7 @@ function RNN:reset(stdv) self.gradWeight:resizeAs(self.weight):zero() end -local function createDescriptors(count, descs_type, create_func, destroy_func) +function RNN:createDescriptors(count, descs_type, create_func, destroy_func) local ds = ffi.new(descs_type, count) for i = 0, count - 1 do errcheck(create_func, ds + i) @@ -64,29 +72,29 @@ local function createDescriptors(count, descs_type, create_func, destroy_func) return ds end -local function createDropoutDescriptors(count) - return createDescriptors(count, +function RNN:createDropoutDescriptors(count) + return self:createDescriptors(count, 'cudnnDropoutDescriptor_t[?]', 'cudnnCreateDropoutDescriptor', 'cudnnDestroyDropoutDescriptor') end -local function createFilterDescriptors(count) - return createDescriptors(count, +function RNN:createFilterDescriptors(count) + return self:createDescriptors(count, 'cudnnFilterDescriptor_t[?]', 'cudnnCreateFilterDescriptor', 'cudnnDestroyFilterDescriptor') end -local function createRNNDescriptors(count) - return createDescriptors(count, +function RNN:createRNNDescriptors(count) + return self:createDescriptors(count, 'cudnnRNNDescriptor_t[?]', 'cudnnCreateRNNDescriptor', 'cudnnDestroyRNNDescriptor') end -local function createTensorDescriptors(count) - return createDescriptors(count, +function RNN:createTensorDescriptors(count) + return self:createDescriptors(count, 'cudnnTensorDescriptor_t[?]', 'cudnnCreateTensorDescriptor', 'cudnnDestroyTensorDescriptor') @@ -94,7 +102,7 @@ end function RNN:resetDropoutDescriptor() if not self.dropoutDesc then - self.dropoutDesc = createDropoutDescriptors(1) + self.dropoutDesc = self:createDropoutDescriptors(1) end self.dropoutStatesSize = torch.LongTensor(1) @@ -113,7 +121,7 @@ end function RNN:resetRNNDescriptor() if not self.rnnDesc then - self.rnnDesc = createRNNDescriptors(1) + self.rnnDesc = self:createRNNDescriptors(1) end errcheck('cudnnSetRNNDescriptor', @@ -130,7 +138,7 @@ end function RNN:resetWeightDescriptor() if not self.wDesc then - self.wDesc = createFilterDescriptors(1) + self.wDesc = self:createFilterDescriptors(1) end local dim = torch.IntTensor({self.weight:size(1), 1, 1}) @@ -144,8 +152,8 @@ function RNN:resetWeightDescriptor() end function RNN:resetIODescriptors() - self.xDescs = createTensorDescriptors(self.seqLength) - self.yDescs = createTensorDescriptors(self.seqLength) + self.xDescs = self:createTensorDescriptors(self.seqLength) + self.yDescs = self:createTensorDescriptors(self.seqLength) for i = 0, self.seqLength - 1 do local dim = torch.IntTensor({self.inputSize, self.miniBatch, self.seqLength}) @@ -169,8 +177,8 @@ function RNN:resetIODescriptors() end function RNN:resetHiddenDescriptors() - self.hxDesc = createTensorDescriptors(1) - self.hyDesc = createTensorDescriptors(1) + self.hxDesc = self:createTensorDescriptors(1) + self.hyDesc = self:createTensorDescriptors(1) local dim = torch.IntTensor({self.hiddenSize, self.miniBatch, self.numLayers}) local stride = torch.IntTensor({1, dim[1], dim[1] * dim[2]}) @@ -190,8 +198,8 @@ function RNN:resetHiddenDescriptors() end function RNN:resetCellDescriptors() - self.cxDesc = createTensorDescriptors(1) - self.cyDesc = createTensorDescriptors(1) + self.cxDesc = self:createTensorDescriptors(1) + self.cyDesc = self:createTensorDescriptors(1) local dim = torch.IntTensor({self.hiddenSize, self.miniBatch, self.numLayers}) local stride = torch.IntTensor({1, dim[1], dim[1] * dim[2]}) @@ -210,7 +218,7 @@ function RNN:resetCellDescriptors() stride:data()) end -local function makeContiguous(self, input, gradOutput) +function RNN:makeContiguous(input, gradOutput) if not input:isContiguous() then self._input = self._input or input.new() self._input:typeAs(input):resizeAs(input):copy(input) @@ -263,11 +271,11 @@ function RNN:updateOutput(input) self:resetWeightDescriptor() end - local x = makeContiguous(self, input) - local y = self.output:resize(self.seqLength, self.miniBatch, self.hiddenSize) + local x = self:makeContiguous(input) + local y = self:resizeOutput(self.output) local w = self.weight - local hy = self.hiddenOutput:resize(self.numLayers, self.miniBatch, self.hiddenSize):zero() - local cy = self.cellOutput:resize(self.numLayers, self.miniBatch, self.hiddenSize):zero() + local hy = self:resizeHidden(self.hiddenOutput):zero() + local cy = self:resizeHidden(self.cellOutput):zero() -- Optionally use hiddenInput/cellInput parameters local hx = self.hiddenInput @@ -351,7 +359,7 @@ function RNN:updateGradInput(input, gradOutput) assert(gradOutput:isSameSizeAs(self.output), 'gradOutput has incorrect size!') assert(self.train, 'updateGradInput can only be called when training!') - local x, dy = makeContiguous(self, input, gradOutput) + local x, dy = self:makeContiguous(self, input, gradOutput) local y = self.output local w = self.weight local dx = self.gradInput:resizeAs(input) @@ -359,8 +367,8 @@ function RNN:updateGradInput(input, gradOutput) local cx = self.cellInput local dhy = self.gradHiddenOutput local dcy = self.gradCellOutput - local dhx = self.gradHiddenInput:resize(self.numLayers, self.miniBatch, self.hiddenSize):zero() - local dcx = self.gradCellInput:resize(self.numLayers, self.miniBatch, self.hiddenSize):zero() + local dhx = self:resizeHidden(self.gradHiddenInput):zero() + local dcx = self:resizeHidden(self.gradCellInput):zero() if hx then @@ -428,7 +436,7 @@ function RNN:accGradParameters(input, gradOutput, scale) assert(gradOutput:isSameSizeAs(self.output), 'gradOutput has incorrect size!') assert(self.train, 'accGradParameters can only be called when training!') - local x, dy = makeContiguous(self, input, gradOutput) + local x, dy = self:makeContiguous(input, gradOutput) local hx = self.hiddenInput local y = self.output local dw = self.gradWeight From dec350f76dd6e58131afd713a764acf9ca8d5a01 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 11 Apr 2016 20:21:53 +0100 Subject: [PATCH 02/49] Added BLSTM based on RNN implementation --- BLSTM.lua | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 BLSTM.lua diff --git a/BLSTM.lua b/BLSTM.lua new file mode 100644 index 0000000..f2853ef --- /dev/null +++ b/BLSTM.lua @@ -0,0 +1,67 @@ +local BLSTM, parent = torch.class('cudnn.BLSTM', 'cudnn.RNN') +local errcheck = cudnn.errcheck + +function BLSTM:__init(inputSize, hiddenSize, numLayers) + parent.__init(self,inputSize, hiddenSize, numLayers) + + self.datatype = 'CUDNN_DATA_FLOAT' + self.inputSize = inputSize + self.hiddenSize = hiddenSize + self.seqLength = 1 + self.miniBatch = 1 + self.numLayers = numLayers + self.bidirectional = 'CUDNN_BIDIRECTIONAL' + self.inputMode = 'CUDNN_LINEAR_INPUT' + self.mode = 'CUDNN_LSTM' + self.dropout = 0 + self.seed = 0x01234567 + + + self.gradInput = torch.CudaTensor() + self.output = torch.CudaTensor() + self.weight = torch.CudaTensor() + self.gradWeight = torch.CudaTensor() + self.reserve = torch.CudaTensor() + self.hiddenOutput = torch.CudaTensor() + self.cellOutput = torch.CudaTensor() + self.gradHiddenInput = torch.CudaTensor() + self.gradCellInput = torch.CudaTensor() + + self:training() + self:reset() + + +end +-- Add * 2 to hidden size since we are implementing a BRNN which concats outputs of both modules. +function BLSTM:resizeOutput(tensor) + return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize * 2) +end + +function BLSTM:resizeHidden(tensor) + return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize * 2) +end + +function BLSTM:resetIODescriptors() + self.xDescs = self:createTensorDescriptors(self.seqLength) + self.yDescs = self:createTensorDescriptors(self.seqLength) + + for i = 0, self.seqLength - 1 do + local dim = torch.IntTensor({ self.inputSize, self.miniBatch, self.seqLength }) + local stride = torch.IntTensor({ 1, dim[1], dim[1] * dim[2] }) + errcheck('cudnnSetTensorNdDescriptor', + self.xDescs[i], + self.datatype, + 3, + dim:data(), + stride:data()) + -- Add * 2 to hidden size since we are implementing a BRNN which concats outputs of both modules. + local dim = torch.IntTensor({ self.hiddenSize * 2, self.miniBatch, self.seqLength }) + local stride = torch.IntTensor({ 1, dim[1], dim[1] * dim[2] }) + errcheck('cudnnSetTensorNdDescriptor', + self.yDescs[i], + self.datatype, + 3, + dim:data(), + stride:data()) + end +end \ No newline at end of file From 9bfed854feabd872fdb153021b5954beff23b59f Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 11 Apr 2016 20:22:11 +0100 Subject: [PATCH 03/49] added BLSTM to init --- init.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/init.lua b/init.lua index d7d2cf7..6528f76 100644 --- a/init.lua +++ b/init.lua @@ -123,6 +123,7 @@ require('cudnn.VolumetricBatchNormalization') require('cudnn.SpatialCrossEntropyCriterion') require('cudnn.TemporalConvolution') require('cudnn.RNN') +require('cudnn.BLSTM') require('cudnn.functional') require('cudnn.convert') From fe29e8b39b810cc0e5324576a4c15e55f4a044fd Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 11 Apr 2016 22:25:33 +0100 Subject: [PATCH 04/49] removed self call in makeContiguous --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index b48bc48..3c0f33d 100644 --- a/RNN.lua +++ b/RNN.lua @@ -359,7 +359,7 @@ function RNN:updateGradInput(input, gradOutput) assert(gradOutput:isSameSizeAs(self.output), 'gradOutput has incorrect size!') assert(self.train, 'updateGradInput can only be called when training!') - local x, dy = self:makeContiguous(self, input, gradOutput) + local x, dy = self:makeContiguous(input, gradOutput) local y = self.output local w = self.weight local dx = self.gradInput:resizeAs(input) From bdc97fab6dd0db73497103130ebc0c1f7d3bd555 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 15:26:52 +0100 Subject: [PATCH 05/49] Added rnn test (for basic RELU) --- test/test_rnn.lua | 178 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 test/test_rnn.lua diff --git a/test/test_rnn.lua b/test/test_rnn.lua new file mode 100644 index 0000000..36624bc --- /dev/null +++ b/test/test_rnn.lua @@ -0,0 +1,178 @@ +require 'cudnn' +require 'cunn' +local ffi = require 'ffi' +local errcheck = cudnn.errcheck + +local cudnntest = torch.TestSuite() +local mytester + +local tolerance = 70 + +function cudnntest.testRNN() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 2 + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers) + + print(string.format("i %E", checkSums.localSumi)) + print(string.format("h %E", checkSums.localSumh)) + print(string.format("c %E", checkSums.localSumc)) + print(string.format("di %E", checkSums.localSumdi)) + print(string.format("dh %E", checkSums.localSumdh)) + print(string.format("dw %E", checkSums.localSumdw)) + + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 1.315212E+05, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumc, 0.000000E+00, tolerance, 'checkSum with reference for localSumc failed') + mytester:assertalmosteq(checkSums.localSumdi, 6.676003E+01, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 6.425067E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 1.453750E+09, tolerance, 'checkSum with reference for localSumdw failed') +end + +--[[ +-- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. +-- ]] +function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers) + + local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn:resetWeightDescriptor() + + -- Matrices are initialised to 1 / matrixSize, biases to 1. + for layer = 0, numberOfLayers - 1 do + for layerId = 0, numberOfLinearLayers - 1 do + local linLayerMatDesc = rnn:createFilterDescriptors(1) + local matrixPointer = ffi.new("float*[1]") + errcheck('cudnnGetRNNLinLayerMatrixParams', + cudnn.getHandle(), + rnn.rnnDesc[0], + layer, + rnn.xDescs, + rnn.wDesc[0], + rnn.weight:data(), + layerId, + linLayerMatDesc[0], + ffi.cast("void**", matrixPointer)) + + local dataType = 'CUDNN_DATA_FLOAT' + local format = 'CUDNN_TENSOR_NCHW' + local nbDims = torch.IntTensor(1) + local filterDimA = torch.IntTensor({ 1, 1, 1 }) + + errcheck('cudnnGetFilterNdDescriptor', + linLayerMatDesc[0], + 3, + ffi.cast("cudnnDataType_t*", dataType), + ffi.cast("cudnnDataType_t*", format), + nbDims:data(), + filterDimA:data()) + + local offset = matrixPointer[0] - rnn.weight:data() + local weightTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA[1] * filterDimA[2] * filterDimA[3]) + weightTensor:fill(1.0 / (filterDimA[1] * filterDimA[2] * filterDimA[3])) + + local linLayerBiasDesc = rnn:createFilterDescriptors(1) + local biasPointer = ffi.new("float*[1]") + errcheck('cudnnGetRNNLinLayerBiasParams', + cudnn.getHandle(), + rnn.rnnDesc[0], + layer, + rnn.xDescs, + rnn.wDesc[0], + rnn.weight:data(), + layerId, + linLayerBiasDesc[0], + ffi.cast("void**", biasPointer)) + + local dataType = 'CUDNN_DATA_FLOAT' + local format = 'CUDNN_TENSOR_NCHW' + local nbDims = torch.IntTensor(1) + local filterDimA = torch.IntTensor({ 1, 1, 1 }) + + errcheck('cudnnGetFilterNdDescriptor', + linLayerBiasDesc[0], + 3, + ffi.cast("cudnnDataType_t*", dataType), + ffi.cast("cudnnDataType_t*", format), + nbDims:data(), + filterDimA:data()) + + local offset = biasPointer[0] - rnn.weight:data() + local biasTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA[1] * filterDimA[2] * filterDimA[3]) + biasTensor:fill(1) + end + end + -- Set hx/cx/dhy/dcy data to 1s. + rnn.hiddenInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) + rnn.cellInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) + rnn.gradHiddenOutput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) + rnn.gradCellOutput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) + + local testOutputi = rnn:forward(input) + -- gradInput set to 1s. + local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) + rnn:backward(input, gradInput) + + -- Sum up all values for each. + local localSumi = 0 + local localSumh = 0 + local localSumc = 0 + for m = 1, seqLength do + for j = 1, miniBatch do + for i = 1, hiddenSize do + localSumi = localSumi + testOutputi[m][j][i] + end + end + end + + for m = 1, numberOfLayers do + for j = 1, miniBatch do + for i = 1, hiddenSize do + localSumh = localSumh + rnn.hiddenOutput[m][j][i] + localSumc = localSumc + rnn.cellOutput[m][j][i] + end + end + end + + local localSumdi = 0 + local localSumdh = 0 + local localSumdc = 0 + for m = 1, seqLength do + for j = 1, miniBatch do + for i = 1, hiddenSize do + localSumdi = localSumdi + rnn.gradInput[m][j][i] + end + end + end + for m = 1, numberOfLayers do + for j = 1, miniBatch do + for i = 1, hiddenSize do + localSumdh = localSumdh + rnn.gradHiddenOutput[m][j][i] + localSumdc = localSumdc + rnn.gradCellOutput[m][j][i] + end + end + end + + local localSumdw = 0 + for m = 1, rnn.gradWeight:size(1) do + localSumdw = localSumdw + rnn.gradWeight[m] + end + + local checkSums = { + localSumi = localSumi, + localSumh = localSumh, + localSumc = localSumc, + localSumdi = localSumdi, + localSumdh = localSumdh, + localSumdc = localSumdc, + localSumdw = localSumdw + } + return checkSums +end + +mytester = torch.Tester() +mytester:add(cudnntest) +mytester:run() \ No newline at end of file From e9de79f5454c7c138d84ad1c2f35b0f7b276f0f7 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 15:28:14 +0100 Subject: [PATCH 06/49] Removed c sum check --- RNN.lua | 3 +-- test/test_rnn.lua | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/RNN.lua b/RNN.lua index 3c0f33d..e7530f3 100644 --- a/RNN.lua +++ b/RNN.lua @@ -1,4 +1,4 @@ -local RNN, parent = torch.class('cudnn.RNN', 'nn.Module') +local RNN, parent = torch.class('cudnn.testRNN', 'nn.Module') local ffi = require 'ffi' local errcheck = cudnn.errcheck @@ -420,7 +420,6 @@ function RNN:updateGradInput(input, gradOutput) self.cxDesc[0], dcx:data(), self.workspace:data(), self.workspace:size(1) * 4, -- sizeof(float) self.reserve:data(), self.reserve:size(1) * 4) -- sizeof(float) - return self.gradInput end diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 36624bc..7cf5b63 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -26,7 +26,6 @@ function cudnntest.testRNN() -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 1.315212E+05, tolerance, 'checkSum with reference for localSumh failed') - mytester:assertalmosteq(checkSums.localSumc, 0.000000E+00, tolerance, 'checkSum with reference for localSumc failed') mytester:assertalmosteq(checkSums.localSumdi, 6.676003E+01, tolerance, 'checkSum with reference for localSumdi failed') mytester:assertalmosteq(checkSums.localSumdh, 6.425067E+01, tolerance, 'checkSum with reference for localSumdh failed') mytester:assertalmosteq(checkSums.localSumdw, 1.453750E+09, tolerance, 'checkSum with reference for localSumdw failed') From 25f53831ed9b49a20f5c73c6004b221ee891037f Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 15:28:30 +0100 Subject: [PATCH 07/49] Removed print statements --- test/test_rnn.lua | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 7cf5b63..3fd5cb8 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -16,13 +16,6 @@ function cudnntest.testRNN() local numberOfLinearLayers = 2 local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers) - print(string.format("i %E", checkSums.localSumi)) - print(string.format("h %E", checkSums.localSumh)) - print(string.format("c %E", checkSums.localSumc)) - print(string.format("di %E", checkSums.localSumdi)) - print(string.format("dh %E", checkSums.localSumdh)) - print(string.format("dw %E", checkSums.localSumdw)) - -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 1.315212E+05, tolerance, 'checkSum with reference for localSumh failed') From d115facb4e4957302684fe48d085ae80f5245fce Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 18:07:22 +0100 Subject: [PATCH 08/49] Fixed call to input grads --- test/test_rnn.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 3fd5cb8..a9361e2 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -142,8 +142,8 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe for m = 1, numberOfLayers do for j = 1, miniBatch do for i = 1, hiddenSize do - localSumdh = localSumdh + rnn.gradHiddenOutput[m][j][i] - localSumdc = localSumdc + rnn.gradCellOutput[m][j][i] + localSumdh = localSumdh + rnn.gradHiddenInput[m][j][i] + localSumdc = localSumdc + rnn.gradCellInput[m][j][i] end end end From 0df09da25ec316c96127131058f8cf8c4fb607b1 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 18:29:45 +0100 Subject: [PATCH 09/49] Added test for all RNN types --- test/test_rnn.lua | 63 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index a9361e2..0e54e02 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -8,13 +8,15 @@ local mytester local tolerance = 70 -function cudnntest.testRNN() + +function cudnntest.testRNNRELU() local miniBatch = 64 local seqLength = 20 local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 2 - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers) + local mode = 'CUDNN_RNN_RELU' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') @@ -24,13 +26,68 @@ function cudnntest.testRNN() mytester:assertalmosteq(checkSums.localSumdw, 1.453750E+09, tolerance, 'checkSum with reference for localSumdw failed') end +function cudnntest.testRNNTANH() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 2 + local mode = 'CUDNN_RNN_TANH' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 6.319591E+05, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 6.319605E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 4.501830E+00, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 4.489546E+00, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 5.012598E+07, tolerance, 'checkSum with reference for localSumdw failed') +end + +function cudnntest.testRNNLSTM() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 8 + local mode = 'CUDNN_LSTM' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 5.749536E+05, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumc, 4.365091E+05, tolerance, 'checkSum with reference for localSumc failed') + mytester:assertalmosteq(checkSums.localSumh, 5.774818E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 3.842206E+02, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdc, 9.323785E+03, tolerance, 'checkSum with reference for localSumdc failed') + mytester:assertalmosteq(checkSums.localSumdh, 1.182566E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 4.313461E+08, tolerance, 'checkSum with reference for localSumdw failed') +end + +function cudnntest.testRNNGRU() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 6 + local mode = 'CUDNN_GRU' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 6.358978E+05, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 6.281680E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 6.296622E+00, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 2.289960E+05, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 5.397419E+07, tolerance, 'checkSum with reference for localSumdw failed') +end + --[[ -- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. -- ]] -function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers) +function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.mode = mode -- Set the mode (GRU/LSTM/ReLU/tanh + rnn:reset() rnn:resetWeightDescriptor() -- Matrices are initialised to 1 / matrixSize, biases to 1. From 11fe8331da8040f1f40a1a1b7966e06057a10c5b Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 18:31:17 +0100 Subject: [PATCH 10/49] Added description --- RNN.lua | 7 ++++++- test/test_rnn.lua | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index e7530f3..442256f 100644 --- a/RNN.lua +++ b/RNN.lua @@ -367,10 +367,10 @@ function RNN:updateGradInput(input, gradOutput) local cx = self.cellInput local dhy = self.gradHiddenOutput local dcy = self.gradCellOutput + local dhx = self:resizeHidden(self.gradHiddenInput):zero() local dcx = self:resizeHidden(self.gradCellInput):zero() - if hx then assert(hx:dim() == 3, 'hiddenInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') assert(hx:size(1) == self.numLayers, 'hiddenInput has incorrect number of layers!') @@ -405,6 +405,8 @@ function RNN:updateGradInput(input, gradOutput) assert(dcy:isContiguous(), 'gradCellOutput must be contiguous!') end + + errcheck('cudnnRNNBackwardData', cudnn.getHandle(), self.rnnDesc[0], @@ -420,6 +422,9 @@ function RNN:updateGradInput(input, gradOutput) self.cxDesc[0], dcx:data(), self.workspace:data(), self.workspace:size(1) * 4, -- sizeof(float) self.reserve:data(), self.reserve:size(1) * 4) -- sizeof(float) + + + return self.gradInput end diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 0e54e02..84c6d1f 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -1,3 +1,7 @@ +--[[ +-- Tests the implementation of RNN binding using the cudnn v5 library. Cross-check the checksums with cudnn reference +-- sample checksums. +-- ]] require 'cudnn' require 'cunn' local ffi = require 'ffi' From 6406b2aa0feb8782b934fe9586068037d0653066 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 18:47:08 +0100 Subject: [PATCH 11/49] Changed tolerance to fix weight difference, small comment change --- test/test_rnn.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 84c6d1f..f4b0a27 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -10,8 +10,7 @@ local errcheck = cudnn.errcheck local cudnntest = torch.TestSuite() local mytester -local tolerance = 70 - +local tolerance = 300 function cudnntest.testRNNRELU() local miniBatch = 64 @@ -90,7 +89,7 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.mode = mode -- Set the mode (GRU/LSTM/ReLU/tanh + rnn.mode = mode -- Set the mode (GRU/LSTM/ReLU/tanh) rnn:reset() rnn:resetWeightDescriptor() From 94c1c22578298a96dce9830643dbf3f36d0eacc0 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 18:52:03 +0100 Subject: [PATCH 12/49] Exposed rnn variable to make easier to replace --- test/test_rnn.lua | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index f4b0a27..b2d625b 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -18,8 +18,9 @@ function cudnntest.testRNNRELU() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 2 - local mode = 'CUDNN_RNN_RELU' - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.mode = 'CUDNN_RNN_RELU' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') @@ -35,8 +36,9 @@ function cudnntest.testRNNTANH() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 2 - local mode = 'CUDNN_RNN_TANH' - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.mode = 'CUDNN_RNN_TANH' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 6.319591E+05, tolerance, 'checkSum with reference for localsumi failed') @@ -52,8 +54,9 @@ function cudnntest.testRNNLSTM() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 8 - local mode = 'CUDNN_LSTM' - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.mode = 'CUDNN_LSTM' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 5.749536E+05, tolerance, 'checkSum with reference for localsumi failed') @@ -71,9 +74,9 @@ function cudnntest.testRNNGRU() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 6 - local mode = 'CUDNN_GRU' - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) - + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.mode = 'CUDNN_GRU' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 6.358978E+05, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 6.281680E+04, tolerance, 'checkSum with reference for localSumh failed') @@ -85,13 +88,11 @@ end --[[ -- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. -- ]] -function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, mode) - - local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.mode = mode -- Set the mode (GRU/LSTM/ReLU/tanh) +function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) + -- Reset the rnn and weight descriptor (since we are manually setting values for matrix/bias. rnn:reset() rnn:resetWeightDescriptor() + local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. -- Matrices are initialised to 1 / matrixSize, biases to 1. for layer = 0, numberOfLayers - 1 do From 18ce0fb33b1af420d538fc224075d08edaa97d94 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 19:21:48 +0100 Subject: [PATCH 13/49] Added base test for BLSTM --- test/test_rnn.lua | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index b2d625b..3117347 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -12,6 +12,24 @@ local mytester local tolerance = 300 +function cudnntest.testBLSTM() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 8 + local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, 2) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 8.178552E+05, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumc, 7.648561E+05, tolerance, 'checkSum with reference for localSumc failed') + mytester:assertalmosteq(checkSums.localSumh, 9.759226E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 6.575394E+02, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdc, 7.621405E+04 , tolerance, 'checkSum with reference for localSumdc failed') + mytester:assertalmosteq(checkSums.localSumdh, 1.451992E+04, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 2.137403E+09, tolerance, 'checkSum with reference for localSumdw failed') +end + function cudnntest.testRNNRELU() local miniBatch = 64 local seqLength = 20 @@ -88,7 +106,8 @@ end --[[ -- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. -- ]] -function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) +function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, biDirectional) + local biDirectionalScale = biDirectional or 1 -- Reset the rnn and weight descriptor (since we are manually setting values for matrix/bias. rnn:reset() rnn:resetWeightDescriptor() @@ -166,7 +185,7 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local testOutputi = rnn:forward(input) -- gradInput set to 1s. - local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) + local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * 2):fill(1) rnn:backward(input, gradInput) -- Sum up all values for each. @@ -175,13 +194,13 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local localSumc = 0 for m = 1, seqLength do for j = 1, miniBatch do - for i = 1, hiddenSize do + for i = 1, hiddenSize * biDirectionalScale do localSumi = localSumi + testOutputi[m][j][i] end end end - for m = 1, numberOfLayers do + for m = 1, numberOfLayers * biDirectionalScale do for j = 1, miniBatch do for i = 1, hiddenSize do localSumh = localSumh + rnn.hiddenOutput[m][j][i] @@ -200,7 +219,7 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe end end end - for m = 1, numberOfLayers do + for m = 1, numberOfLayers * biDirectionalScale do for j = 1, miniBatch do for i = 1, hiddenSize do localSumdh = localSumdh + rnn.gradHiddenInput[m][j][i] From 291bd5060f9d31797de7bfdd8a01b70d9c4bfc89 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 19:22:01 +0100 Subject: [PATCH 14/49] Revert "Added description" This reverts commit 11fe8331da8040f1f40a1a1b7966e06057a10c5b. --- RNN.lua | 7 +------ test/test_rnn.lua | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/RNN.lua b/RNN.lua index 442256f..e7530f3 100644 --- a/RNN.lua +++ b/RNN.lua @@ -367,10 +367,10 @@ function RNN:updateGradInput(input, gradOutput) local cx = self.cellInput local dhy = self.gradHiddenOutput local dcy = self.gradCellOutput - local dhx = self:resizeHidden(self.gradHiddenInput):zero() local dcx = self:resizeHidden(self.gradCellInput):zero() + if hx then assert(hx:dim() == 3, 'hiddenInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') assert(hx:size(1) == self.numLayers, 'hiddenInput has incorrect number of layers!') @@ -405,8 +405,6 @@ function RNN:updateGradInput(input, gradOutput) assert(dcy:isContiguous(), 'gradCellOutput must be contiguous!') end - - errcheck('cudnnRNNBackwardData', cudnn.getHandle(), self.rnnDesc[0], @@ -422,9 +420,6 @@ function RNN:updateGradInput(input, gradOutput) self.cxDesc[0], dcx:data(), self.workspace:data(), self.workspace:size(1) * 4, -- sizeof(float) self.reserve:data(), self.reserve:size(1) * 4) -- sizeof(float) - - - return self.gradInput end diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 3117347..8c657f9 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -1,7 +1,3 @@ ---[[ --- Tests the implementation of RNN binding using the cudnn v5 library. Cross-check the checksums with cudnn reference --- sample checksums. --- ]] require 'cudnn' require 'cunn' local ffi = require 'ffi' From b09c18499057a4b901c99ebd8f29e58b2dba370b Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 19:25:34 +0100 Subject: [PATCH 15/49] Fixed change of RNN module name --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index e7530f3..10f2cca 100644 --- a/RNN.lua +++ b/RNN.lua @@ -1,4 +1,4 @@ -local RNN, parent = torch.class('cudnn.testRNN', 'nn.Module') +local RNN, parent = torch.class('cudnn.RNN', 'nn.Module') local ffi = require 'ffi' local errcheck = cudnn.errcheck From 12a769622a087c2dc92b01c4c9e0bd221fe02702 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 19:48:00 +0100 Subject: [PATCH 16/49] Added comment removed BLSTM test --- test/test_rnn.lua | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 8c657f9..66fdaf4 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -1,3 +1,8 @@ +--[[ +-- Tests the implementation of RNN binding using the cudnn v5 library. Cross-check the checksums with cudnn reference +-- sample checksums. +-- ]] + require 'cudnn' require 'cunn' local ffi = require 'ffi' @@ -8,24 +13,6 @@ local mytester local tolerance = 300 -function cudnntest.testBLSTM() - local miniBatch = 64 - local seqLength = 20 - local hiddenSize = 512 - local numberOfLayers = 2 - local numberOfLinearLayers = 8 - local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, 2) - -- Checksums to check against are retrieved from cudnn RNN sample. - mytester:assertalmosteq(checkSums.localSumi, 8.178552E+05, tolerance, 'checkSum with reference for localsumi failed') - mytester:assertalmosteq(checkSums.localSumc, 7.648561E+05, tolerance, 'checkSum with reference for localSumc failed') - mytester:assertalmosteq(checkSums.localSumh, 9.759226E+04, tolerance, 'checkSum with reference for localSumh failed') - mytester:assertalmosteq(checkSums.localSumdi, 6.575394E+02, tolerance, 'checkSum with reference for localSumdi failed') - mytester:assertalmosteq(checkSums.localSumdc, 7.621405E+04 , tolerance, 'checkSum with reference for localSumdc failed') - mytester:assertalmosteq(checkSums.localSumdh, 1.451992E+04, tolerance, 'checkSum with reference for localSumdh failed') - mytester:assertalmosteq(checkSums.localSumdw, 2.137403E+09, tolerance, 'checkSum with reference for localSumdw failed') -end - function cudnntest.testRNNRELU() local miniBatch = 64 local seqLength = 20 From 128725090d3a284137a5eb09924cacf74c067b30 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 20:29:46 +0100 Subject: [PATCH 17/49] Added numDirections param to RNN --- RNN.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/RNN.lua b/RNN.lua index 10f2cca..48ca4f5 100644 --- a/RNN.lua +++ b/RNN.lua @@ -12,6 +12,7 @@ function RNN:__init(inputSize, hiddenSize, numLayers) self.miniBatch = 1 self.numLayers = numLayers self.bidirectional = 'CUDNN_UNIDIRECTIONAL' + self.numDirections = 1 -- set to 2 for bi-directional. self.inputMode = 'CUDNN_LINEAR_INPUT' self.mode = 'CUDNN_RNN_RELU' self.dropout = 0 @@ -32,11 +33,11 @@ function RNN:__init(inputSize, hiddenSize, numLayers) end function RNN:resizeOutput(tensor) - return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize) + return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize * self.numDirections) end function RNN:resizeHidden(tensor) - return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize) + return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize * self.numDirections) end function RNN:reset(stdv) @@ -165,7 +166,7 @@ function RNN:resetIODescriptors() dim:data(), stride:data()) - local dim = torch.IntTensor({self.hiddenSize, self.miniBatch, self.seqLength}) + local dim = torch.IntTensor({self.hiddenSize * self.numDirections, self.miniBatch, self.seqLength}) local stride = torch.IntTensor({1, dim[1], dim[1] * dim[2]}) errcheck('cudnnSetTensorNdDescriptor', self.yDescs[i], From c98e24f92092c94d65776d5b8f5b0a785f769721 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 20:31:03 +0100 Subject: [PATCH 18/49] Added numDirections param instead of duplicated methods --- BLSTM.lua | 60 +------------------------------------------------------ 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/BLSTM.lua b/BLSTM.lua index f2853ef..e452cb6 100644 --- a/BLSTM.lua +++ b/BLSTM.lua @@ -1,67 +1,9 @@ local BLSTM, parent = torch.class('cudnn.BLSTM', 'cudnn.RNN') -local errcheck = cudnn.errcheck function BLSTM:__init(inputSize, hiddenSize, numLayers) parent.__init(self,inputSize, hiddenSize, numLayers) - - self.datatype = 'CUDNN_DATA_FLOAT' - self.inputSize = inputSize - self.hiddenSize = hiddenSize - self.seqLength = 1 - self.miniBatch = 1 - self.numLayers = numLayers self.bidirectional = 'CUDNN_BIDIRECTIONAL' - self.inputMode = 'CUDNN_LINEAR_INPUT' self.mode = 'CUDNN_LSTM' - self.dropout = 0 - self.seed = 0x01234567 - - - self.gradInput = torch.CudaTensor() - self.output = torch.CudaTensor() - self.weight = torch.CudaTensor() - self.gradWeight = torch.CudaTensor() - self.reserve = torch.CudaTensor() - self.hiddenOutput = torch.CudaTensor() - self.cellOutput = torch.CudaTensor() - self.gradHiddenInput = torch.CudaTensor() - self.gradCellInput = torch.CudaTensor() - - self:training() + self.numDirections = 2 self:reset() - - end --- Add * 2 to hidden size since we are implementing a BRNN which concats outputs of both modules. -function BLSTM:resizeOutput(tensor) - return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize * 2) -end - -function BLSTM:resizeHidden(tensor) - return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize * 2) -end - -function BLSTM:resetIODescriptors() - self.xDescs = self:createTensorDescriptors(self.seqLength) - self.yDescs = self:createTensorDescriptors(self.seqLength) - - for i = 0, self.seqLength - 1 do - local dim = torch.IntTensor({ self.inputSize, self.miniBatch, self.seqLength }) - local stride = torch.IntTensor({ 1, dim[1], dim[1] * dim[2] }) - errcheck('cudnnSetTensorNdDescriptor', - self.xDescs[i], - self.datatype, - 3, - dim:data(), - stride:data()) - -- Add * 2 to hidden size since we are implementing a BRNN which concats outputs of both modules. - local dim = torch.IntTensor({ self.hiddenSize * 2, self.miniBatch, self.seqLength }) - local stride = torch.IntTensor({ 1, dim[1], dim[1] * dim[2] }) - errcheck('cudnnSetTensorNdDescriptor', - self.yDescs[i], - self.datatype, - 3, - dim:data(), - stride:data()) - end -end \ No newline at end of file From 4d83082854a738b6d2dc345caa04ca0882e42dab Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 20:32:34 +0100 Subject: [PATCH 19/49] Removed *2 on gradInput --- test/test_rnn.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 66fdaf4..efd337b 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -168,7 +168,7 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local testOutputi = rnn:forward(input) -- gradInput set to 1s. - local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * 2):fill(1) + local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) rnn:backward(input, gradInput) -- Sum up all values for each. From 3c13e7efc59f7bf48f3d50fec06884b08c1cb4a5 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 20:38:36 +0100 Subject: [PATCH 20/49] Removed hardcoded 3 dimension from cudnn call --- test/test_rnn.lua | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index efd337b..d711067 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -115,19 +115,20 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local dataType = 'CUDNN_DATA_FLOAT' local format = 'CUDNN_TENSOR_NCHW' local nbDims = torch.IntTensor(1) - local filterDimA = torch.IntTensor({ 1, 1, 1 }) + local minDim = 3 + local filterDimA = torch.ones(minDim):int() errcheck('cudnnGetFilterNdDescriptor', linLayerMatDesc[0], - 3, + minDim, ffi.cast("cudnnDataType_t*", dataType), ffi.cast("cudnnDataType_t*", format), nbDims:data(), filterDimA:data()) local offset = matrixPointer[0] - rnn.weight:data() - local weightTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA[1] * filterDimA[2] * filterDimA[3]) - weightTensor:fill(1.0 / (filterDimA[1] * filterDimA[2] * filterDimA[3])) + local weightTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) + weightTensor:fill(1.0 / filterDimA:prod()) local linLayerBiasDesc = rnn:createFilterDescriptors(1) local biasPointer = ffi.new("float*[1]") @@ -145,18 +146,18 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local dataType = 'CUDNN_DATA_FLOAT' local format = 'CUDNN_TENSOR_NCHW' local nbDims = torch.IntTensor(1) - local filterDimA = torch.IntTensor({ 1, 1, 1 }) + local filterDimA = torch.ones(minDim):int() errcheck('cudnnGetFilterNdDescriptor', linLayerBiasDesc[0], - 3, + minDim, ffi.cast("cudnnDataType_t*", dataType), ffi.cast("cudnnDataType_t*", format), nbDims:data(), filterDimA:data()) local offset = biasPointer[0] - rnn.weight:data() - local biasTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA[1] * filterDimA[2] * filterDimA[3]) + local biasTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) biasTensor:fill(1) end end From e80fce1b96b07dc94de585f64002e8e9a8e17548 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Wed, 13 Apr 2016 20:49:11 +0100 Subject: [PATCH 21/49] Added BLSTM test --- test/test_rnn.lua | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index d711067..ae761a3 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -13,6 +13,26 @@ local mytester local tolerance = 300 + +function cudnntest.testBLSTM() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 8 + local nbDirections = 2 + local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 5.749536E+05, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumc, 4.365091E+05, tolerance, 'checkSum with reference for localSumc failed') + mytester:assertalmosteq(checkSums.localSumh, 5.774818E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 3.842206E+02, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdc, 9.323785E+03, tolerance, 'checkSum with reference for localSumdc failed') + mytester:assertalmosteq(checkSums.localSumdh, 1.182566E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 4.313461E+08, tolerance, 'checkSum with reference for localSumdw failed') +end + function cudnntest.testRNNRELU() local miniBatch = 64 local seqLength = 20 @@ -162,14 +182,14 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe end end -- Set hx/cx/dhy/dcy data to 1s. - rnn.hiddenInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) - rnn.cellInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) - rnn.gradHiddenOutput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) - rnn.gradCellOutput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize):fill(1) + rnn.hiddenInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize * biDirectionalScale):fill(1) + rnn.cellInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize * biDirectionalScale):fill(1) + rnn.gradHiddenOutput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) + rnn.gradCellOutput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) local testOutputi = rnn:forward(input) -- gradInput set to 1s. - local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) + local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * biDirectionalScale):fill(1) rnn:backward(input, gradInput) -- Sum up all values for each. From 54488bbfef6e21f05c95ec32cc8e181248d56219 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 14 Apr 2016 19:49:32 +0100 Subject: [PATCH 22/49] Fixed resize of hidden tensors --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index 48ca4f5..fbb02a1 100644 --- a/RNN.lua +++ b/RNN.lua @@ -37,7 +37,7 @@ function RNN:resizeOutput(tensor) end function RNN:resizeHidden(tensor) - return tensor:resize(self.numLayers, self.miniBatch, self.hiddenSize * self.numDirections) + return tensor:resize(self.numLayers * self.numDirections, self.miniBatch, self.hiddenSize) end function RNN:reset(stdv) From 9e58334626c946ca156c0e7c162434870cf8491b Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 14 Apr 2016 19:52:27 +0100 Subject: [PATCH 23/49] Added numDirections to assertions --- RNN.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/RNN.lua b/RNN.lua index fbb02a1..6a24293 100644 --- a/RNN.lua +++ b/RNN.lua @@ -284,14 +284,14 @@ function RNN:updateOutput(input) if hx then assert(hx:dim() == 3, 'hiddenInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') - assert(hx:size(1) == self.numLayers, 'hiddenInput has incorrect number of layers!') + assert(hx:size(1) == self.numLayers * self.numDirections, 'hiddenInput has incorrect number of layers!') assert(hx:size(2) == self.miniBatch, 'hiddenInput has incorrect number of minibathes!') assert(hx:size(3) == self.hiddenSize, 'hiddenIinput has incorrect size!') assert(hx:isContiguous(), 'hiddenInput must be contiguous!') end if cx then assert(cx:dim() == 3, 'cellInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') - assert(cx:size(1) == self.numLayers, 'cellInput has incorrect number of layers!') + assert(cx:size(1) == self.numLayers * self.numDirections, 'cellInput has incorrect number of layers!') assert(cx:size(2) == self.miniBatch, 'cellInput has incorrect number of minibathes!') assert(cx:size(3) == self.hiddenSize, 'cellInput has incorrect size!') assert(cx:isContiguous(), 'cellInput must be contiguous!') @@ -374,7 +374,7 @@ function RNN:updateGradInput(input, gradOutput) if hx then assert(hx:dim() == 3, 'hiddenInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') - assert(hx:size(1) == self.numLayers, 'hiddenInput has incorrect number of layers!') + assert(hx:size(1) == self.numLayers * self.numDirections, 'hiddenInput has incorrect number of layers!') assert(hx:size(2) == self.miniBatch, 'hiddenInput has incorrect minibatch size!') assert(hx:size(3) == self.hiddenSize, 'hiddenInput has incorrect size!') assert(hx:isContiguous(), 'hiddenInput must be contiguous!') @@ -382,7 +382,7 @@ function RNN:updateGradInput(input, gradOutput) if cx then assert(cx:dim() == 3, 'cellInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') - assert(cx:size(1) == self.numLayers, 'cellInput has incorrect number of layers!') + assert(cx:size(1) == self.numLayers * self.numDirections, 'cellInput has incorrect number of layers!') assert(cx:size(2) == self.miniBatch, 'cellInput has incorrect minibatch size!') assert(cx:size(3) == self.hiddenSize, 'cellInput has incorrect size!') assert(cx:isContiguous(), 'cellInput must be contiguous!') @@ -391,7 +391,7 @@ function RNN:updateGradInput(input, gradOutput) if dhy then assert(dhy:dim() == 3, 'gradHiddenOutput must have 3 dimensions: ' .. 'numLayers, miniBatch, hiddenSize') - assert(dhy:size(1) == self.numLayers, 'gradHiddenOutput has incorrect number of layers!') + assert(dhy:size(1) == self.numLayers * self.numDirections, 'gradHiddenOutput has incorrect number of layers!') assert(dhy:size(2) == self.miniBatch, 'gradHiddenOutput has incorrect minibatch size!') assert(dhy:size(3) == self.hiddenSize, 'gradHiddenOutput has incorrect size!') assert(dhy:isContiguous(), 'gradHiddenOutput must be contiguous!') @@ -400,7 +400,7 @@ function RNN:updateGradInput(input, gradOutput) if dcy then assert(dcy:dim() == 3, 'gradCellOutput must have 3 dimensions: ' .. 'numLayers, miniBatch, hiddenSize') - assert(dcy:size(1) == self.numLayers, 'gradCellOutput has incorrect number of layers!') + assert(dcy:size(1) == self.numLayers * self.numDirections, 'gradCellOutput has incorrect number of layers!') assert(dcy:size(2) == self.miniBatch, 'gradCellOutput has incorrect minibatch size!') assert(dcy:size(3) == self.hiddenSize, 'gradCellOutput has incorrect size!') assert(dcy:isContiguous(), 'gradCellOutput must be contiguous!') From 04feef34cf8e7895afb117b8ec5ce6627a5ff2ed Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 14 Apr 2016 19:55:20 +0100 Subject: [PATCH 24/49] Added assertion check to accGradParams --- RNN.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RNN.lua b/RNN.lua index 6a24293..c57e790 100644 --- a/RNN.lua +++ b/RNN.lua @@ -1,4 +1,4 @@ -local RNN, parent = torch.class('cudnn.RNN', 'nn.Module') +local RNN, parent = torch.class('cudnn.testRNN', 'nn.Module') local ffi = require 'ffi' local errcheck = cudnn.errcheck @@ -443,7 +443,7 @@ function RNN:accGradParameters(input, gradOutput, scale) if hx then assert(hx:dim() == 3, 'hiddenInput must have 3 dimensions: numLayers, miniBatch, hiddenSize') - assert(hx:size(1) == self.numLayers, 'hiddenInput has incorrect number of layers!') + assert(hx:size(1) == self.numLayers * self.numDirections, 'hiddenInput has incorrect number of layers!') assert(hx:size(2) == self.miniBatch, 'hiddenInput has incorrect minibatch size!') assert(hx:size(3) == self.hiddenSize, 'hiddenIinput has incorrect size!') assert(hx:isContiguous(), 'hiddenInput must be contiguous!') From 9d443930b36049a43ceb44b7c8ad4698bf7c571f Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 14 Apr 2016 23:11:41 +0100 Subject: [PATCH 25/49] Added BRNN test --- test/test_rnn.lua | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index ae761a3..1b50d1d 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -13,24 +13,25 @@ local mytester local tolerance = 300 - -function cudnntest.testBLSTM() +function cudnntest.testBiDirectionalRNN() local miniBatch = 64 local seqLength = 20 local hiddenSize = 512 local numberOfLayers = 2 - local numberOfLinearLayers = 8 + local numberOfLinearLayers = 2 local nbDirections = 2 - local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' + rnn.mode = 'CUDNN_RNN_TANH' + rnn.numDirections = 2 + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. - mytester:assertalmosteq(checkSums.localSumi, 5.749536E+05, tolerance, 'checkSum with reference for localsumi failed') - mytester:assertalmosteq(checkSums.localSumc, 4.365091E+05, tolerance, 'checkSum with reference for localSumc failed') - mytester:assertalmosteq(checkSums.localSumh, 5.774818E+04, tolerance, 'checkSum with reference for localSumh failed') - mytester:assertalmosteq(checkSums.localSumdi, 3.842206E+02, tolerance, 'checkSum with reference for localSumdi failed') - mytester:assertalmosteq(checkSums.localSumdc, 9.323785E+03, tolerance, 'checkSum with reference for localSumdc failed') - mytester:assertalmosteq(checkSums.localSumdh, 1.182566E+01, tolerance, 'checkSum with reference for localSumdh failed') - mytester:assertalmosteq(checkSums.localSumdw, 4.313461E+08, tolerance, 'checkSum with reference for localSumdw failed') + mytester:assertalmosteq(checkSums.localSumi, 4.427218E+03, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 6.340836E+04, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 7.226166E+00, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 9.618800E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 1.404362E+09, tolerance, 'checkSum with reference for localSumdw failed') end function cudnntest.testRNNRELU() @@ -182,11 +183,10 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe end end -- Set hx/cx/dhy/dcy data to 1s. - rnn.hiddenInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize * biDirectionalScale):fill(1) - rnn.cellInput = torch.CudaTensor(numberOfLayers, miniBatch, hiddenSize * biDirectionalScale):fill(1) + rnn.hiddenInput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) + rnn.cellInput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) rnn.gradHiddenOutput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) rnn.gradCellOutput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) - local testOutputi = rnn:forward(input) -- gradInput set to 1s. local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * biDirectionalScale):fill(1) From 1cdff2137b7bf51c2f0f969d59e5f365dd7c26fa Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Thu, 14 Apr 2016 23:14:28 +0100 Subject: [PATCH 26/49] Reverted change of class name --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index c57e790..400c978 100644 --- a/RNN.lua +++ b/RNN.lua @@ -1,4 +1,4 @@ -local RNN, parent = torch.class('cudnn.testRNN', 'nn.Module') +local RNN, parent = torch.class('cudnn.RNN', 'nn.Module') local ffi = require 'ffi' local errcheck = cudnn.errcheck From d8d4f89ed04db00bd7b64d04aad540aa27607962 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 17:33:32 +0100 Subject: [PATCH 27/49] Added ReLU/tanh/LSTM/GRU bidirectional tests --- test/test_rnn.lua | 237 ++++++++++++++++++++++++++++++---------------- 1 file changed, 153 insertions(+), 84 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 1b50d1d..4dcb246 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -13,27 +13,6 @@ local mytester local tolerance = 300 -function cudnntest.testBiDirectionalRNN() - local miniBatch = 64 - local seqLength = 20 - local hiddenSize = 512 - local numberOfLayers = 2 - local numberOfLinearLayers = 2 - local nbDirections = 2 - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' - rnn.mode = 'CUDNN_RNN_TANH' - rnn.numDirections = 2 - - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) - -- Checksums to check against are retrieved from cudnn RNN sample. - mytester:assertalmosteq(checkSums.localSumi, 4.427218E+03, tolerance, 'checkSum with reference for localsumi failed') - mytester:assertalmosteq(checkSums.localSumh, 6.340836E+04, tolerance, 'checkSum with reference for localSumh failed') - mytester:assertalmosteq(checkSums.localSumdi, 7.226166E+00, tolerance, 'checkSum with reference for localSumdi failed') - mytester:assertalmosteq(checkSums.localSumdh, 9.618800E+01, tolerance, 'checkSum with reference for localSumdh failed') - mytester:assertalmosteq(checkSums.localSumdw, 1.404362E+09, tolerance, 'checkSum with reference for localSumdw failed') -end - function cudnntest.testRNNRELU() local miniBatch = 64 local seqLength = 20 @@ -107,6 +86,92 @@ function cudnntest.testRNNGRU() mytester:assertalmosteq(checkSums.localSumdw, 5.397419E+07, tolerance, 'checkSum with reference for localSumdw failed') end +function cudnntest.testBiDirectionalRELURNN() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 2 + local nbDirections = 2 + local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' + rnn.mode = 'CUDNN_RNN_RELU' + rnn.numDirections = 2 + + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 1.388634E+01, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 1.288997E+01, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 1.288729E+01, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 1.279004E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 7.061081E+07, tolerance, 'checkSum with reference for localSumdw failed') +end + +function cudnntest.testBiDirectionalTANHRNN() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 2 + local nbDirections = 2 + local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' + rnn.mode = 'CUDNN_RNN_TANH' + rnn.numDirections = 2 + + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 1.388634E+01, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 1.288997E+01, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 1.288729E+01, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 1.279004E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 7.061081E+07, tolerance, 'checkSum with reference for localSumdw failed') +end + +function cudnntest.testBiDirectionalLSTMRNN() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 8 + local nbDirections = 2 + local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' + rnn.mode = 'CUDNN_LSTM' + rnn.numDirections = 2 + + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 3.134097E+04, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumc, 3.845626E+00, tolerance, 'checkSum with reference for localSumc failed') + mytester:assertalmosteq(checkSums.localSumh, 1.922855E+00, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 4.794993E+00, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdc, 2.870925E+04, tolerance, 'checkSum with reference for localSumdc failed') + mytester:assertalmosteq(checkSums.localSumdh, 2.468645E+00, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 1.121568E+08, tolerance, 'checkSum with reference for localSumdw failed') +end + +function cudnntest.testBiDirectionalGRURNN() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 6 + local nbDirections = 2 + local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' + rnn.mode = 'CUDNN_GRU' + rnn.numDirections = 2 + + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 6.555183E+04, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 5.830924E+00, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 4.271801E+00, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 6.555744E+04, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 1.701796E+08, tolerance, 'checkSum with reference for localSumdw failed') +end + --[[ -- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. -- ]] @@ -117,69 +182,73 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe rnn:resetWeightDescriptor() local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. - -- Matrices are initialised to 1 / matrixSize, biases to 1. - for layer = 0, numberOfLayers - 1 do - for layerId = 0, numberOfLinearLayers - 1 do - local linLayerMatDesc = rnn:createFilterDescriptors(1) - local matrixPointer = ffi.new("float*[1]") - errcheck('cudnnGetRNNLinLayerMatrixParams', - cudnn.getHandle(), - rnn.rnnDesc[0], - layer, - rnn.xDescs, - rnn.wDesc[0], - rnn.weight:data(), - layerId, - linLayerMatDesc[0], - ffi.cast("void**", matrixPointer)) - - local dataType = 'CUDNN_DATA_FLOAT' - local format = 'CUDNN_TENSOR_NCHW' - local nbDims = torch.IntTensor(1) - - local minDim = 3 - local filterDimA = torch.ones(minDim):int() - errcheck('cudnnGetFilterNdDescriptor', - linLayerMatDesc[0], - minDim, - ffi.cast("cudnnDataType_t*", dataType), - ffi.cast("cudnnDataType_t*", format), - nbDims:data(), - filterDimA:data()) - - local offset = matrixPointer[0] - rnn.weight:data() - local weightTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) - weightTensor:fill(1.0 / filterDimA:prod()) - - local linLayerBiasDesc = rnn:createFilterDescriptors(1) - local biasPointer = ffi.new("float*[1]") - errcheck('cudnnGetRNNLinLayerBiasParams', - cudnn.getHandle(), - rnn.rnnDesc[0], - layer, - rnn.xDescs, - rnn.wDesc[0], - rnn.weight:data(), - layerId, - linLayerBiasDesc[0], - ffi.cast("void**", biasPointer)) - - local dataType = 'CUDNN_DATA_FLOAT' - local format = 'CUDNN_TENSOR_NCHW' - local nbDims = torch.IntTensor(1) - local filterDimA = torch.ones(minDim):int() - - errcheck('cudnnGetFilterNdDescriptor', - linLayerBiasDesc[0], - minDim, - ffi.cast("cudnnDataType_t*", dataType), - ffi.cast("cudnnDataType_t*", format), - nbDims:data(), - filterDimA:data()) - - local offset = biasPointer[0] - rnn.weight:data() - local biasTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) - biasTensor:fill(1) + if (biDirectional) then + rnn.weight:fill(1 / rnn.weight:size(1)) + else + -- Matrices are initialised to 1 / matrixSize, biases to 1. + for layer = 0, numberOfLayers - 1 do + for layerId = 0, numberOfLinearLayers - 1 do + local linLayerMatDesc = rnn:createFilterDescriptors(1) + local matrixPointer = ffi.new("float*[1]") + errcheck('cudnnGetRNNLinLayerMatrixParams', + cudnn.getHandle(), + rnn.rnnDesc[0], + layer, + rnn.xDescs, + rnn.wDesc[0], + rnn.weight:data(), + layerId, + linLayerMatDesc[0], + ffi.cast("void**", matrixPointer)) + + local dataType = 'CUDNN_DATA_FLOAT' + local format = 'CUDNN_TENSOR_NCHW' + local nbDims = torch.IntTensor(1) + + local minDim = 3 + local filterDimA = torch.ones(minDim):int() + errcheck('cudnnGetFilterNdDescriptor', + linLayerMatDesc[0], + minDim, + ffi.cast("cudnnDataType_t*", dataType), + ffi.cast("cudnnDataType_t*", format), + nbDims:data(), + filterDimA:data()) + + local offset = matrixPointer[0] - rnn.weight:data() + local weightTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) + weightTensor:fill(1.0 / filterDimA:prod()) + + local linLayerBiasDesc = rnn:createFilterDescriptors(1) + local biasPointer = ffi.new("float*[1]") + errcheck('cudnnGetRNNLinLayerBiasParams', + cudnn.getHandle(), + rnn.rnnDesc[0], + layer, + rnn.xDescs, + rnn.wDesc[0], + rnn.weight:data(), + layerId, + linLayerBiasDesc[0], + ffi.cast("void**", biasPointer)) + + local dataType = 'CUDNN_DATA_FLOAT' + local format = 'CUDNN_TENSOR_NCHW' + local nbDims = torch.IntTensor(1) + local filterDimA = torch.ones(minDim):int() + + errcheck('cudnnGetFilterNdDescriptor', + linLayerBiasDesc[0], + minDim, + ffi.cast("cudnnDataType_t*", dataType), + ffi.cast("cudnnDataType_t*", format), + nbDims:data(), + filterDimA:data()) + + local offset = biasPointer[0] - rnn.weight:data() + local biasTensor = torch.CudaTensor(rnn.weight:storage(), offset + 1, filterDimA:prod()) + biasTensor:fill(1) + end end end -- Set hx/cx/dhy/dcy data to 1s. From ea21e9d3dcb567b935cb5770b145b439370a9caf Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:42:38 +0100 Subject: [PATCH 28/49] Updated modules in tests --- test/test_rnn.lua | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 4dcb246..82a72ee 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -55,8 +55,7 @@ function cudnntest.testRNNLSTM() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 8 - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.mode = 'CUDNN_LSTM' + local rnn = cudnn.LSTM(hiddenSize, hiddenSize, numberOfLayers) local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. @@ -75,8 +74,7 @@ function cudnntest.testRNNGRU() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 6 - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.mode = 'CUDNN_GRU' + local rnn = cudnn.GRU(hiddenSize, hiddenSize, numberOfLayers) local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 6.358978E+05, tolerance, 'checkSum with reference for localsumi failed') @@ -93,7 +91,7 @@ function cudnntest.testBiDirectionalRELURNN() local numberOfLayers = 2 local numberOfLinearLayers = 2 local nbDirections = 2 - local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_RNN_RELU' rnn.numDirections = 2 @@ -114,7 +112,7 @@ function cudnntest.testBiDirectionalTANHRNN() local numberOfLayers = 2 local numberOfLinearLayers = 2 local nbDirections = 2 - local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_RNN_TANH' rnn.numDirections = 2 @@ -135,10 +133,7 @@ function cudnntest.testBiDirectionalLSTMRNN() local numberOfLayers = 2 local numberOfLinearLayers = 8 local nbDirections = 2 - local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) - rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' - rnn.mode = 'CUDNN_LSTM' - rnn.numDirections = 2 + local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. @@ -158,7 +153,7 @@ function cudnntest.testBiDirectionalGRURNN() local numberOfLayers = 2 local numberOfLinearLayers = 6 local nbDirections = 2 - local rnn = cudnn.testRNN(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_GRU' rnn.numDirections = 2 From 31f5f1737ff7d271c195ed33a7724967d36b8ebb Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:42:55 +0100 Subject: [PATCH 29/49] Added GRU and LSTM --- GRU.lua | 8 ++++++++ LSTM.lua | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 GRU.lua create mode 100644 LSTM.lua diff --git a/GRU.lua b/GRU.lua new file mode 100644 index 0000000..4670874 --- /dev/null +++ b/GRU.lua @@ -0,0 +1,8 @@ +local GRU, parent = torch.class('cudnn.GRU', 'cudnn.RNN') + +function BLSTM:__init(inputSize, hiddenSize, numLayers) + parent.__init(self,inputSize, hiddenSize, numLayers) + self.bidirectional = 'CUDNN_UNIDIRECTIONAL' + self.mode = 'CUDNN_GRU' + self:reset() +end diff --git a/LSTM.lua b/LSTM.lua new file mode 100644 index 0000000..cbc5970 --- /dev/null +++ b/LSTM.lua @@ -0,0 +1,8 @@ +local BLSTM, parent = torch.class('cudnn.LSTM', 'cudnn.RNN') + +function BLSTM:__init(inputSize, hiddenSize, numLayers) + parent.__init(self,inputSize, hiddenSize, numLayers) + self.bidirectional = 'CUDNN_UNIDIRECTIONAL' + self.mode = 'CUDNN_LSTM' + self:reset() +end From 242d61320e5b9118554af815658b45d8baae33ed Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:43:52 +0100 Subject: [PATCH 30/49] Added GRU/LSTM to init --- init.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init.lua b/init.lua index 6528f76..097600f 100644 --- a/init.lua +++ b/init.lua @@ -123,6 +123,8 @@ require('cudnn.VolumetricBatchNormalization') require('cudnn.SpatialCrossEntropyCriterion') require('cudnn.TemporalConvolution') require('cudnn.RNN') +require('cudnn.LSTM') +require('cudnn.GRU') require('cudnn.BLSTM') require('cudnn.functional') require('cudnn.convert') From 170e90d3010339f2d57f42a617ba863eae7b9599 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:47:13 +0100 Subject: [PATCH 31/49] Removed direction set --- GRU.lua | 1 - LSTM.lua | 1 - 2 files changed, 2 deletions(-) diff --git a/GRU.lua b/GRU.lua index 4670874..73c6ed1 100644 --- a/GRU.lua +++ b/GRU.lua @@ -2,7 +2,6 @@ local GRU, parent = torch.class('cudnn.GRU', 'cudnn.RNN') function BLSTM:__init(inputSize, hiddenSize, numLayers) parent.__init(self,inputSize, hiddenSize, numLayers) - self.bidirectional = 'CUDNN_UNIDIRECTIONAL' self.mode = 'CUDNN_GRU' self:reset() end diff --git a/LSTM.lua b/LSTM.lua index cbc5970..096fd6b 100644 --- a/LSTM.lua +++ b/LSTM.lua @@ -2,7 +2,6 @@ local BLSTM, parent = torch.class('cudnn.LSTM', 'cudnn.RNN') function BLSTM:__init(inputSize, hiddenSize, numLayers) parent.__init(self,inputSize, hiddenSize, numLayers) - self.bidirectional = 'CUDNN_UNIDIRECTIONAL' self.mode = 'CUDNN_LSTM' self:reset() end From 2b82dc048144edc65bfd15f133ca60cc10be7719 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:47:26 +0100 Subject: [PATCH 32/49] Changed init --- init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init.lua b/init.lua index 097600f..9ff687a 100644 --- a/init.lua +++ b/init.lua @@ -123,9 +123,9 @@ require('cudnn.VolumetricBatchNormalization') require('cudnn.SpatialCrossEntropyCriterion') require('cudnn.TemporalConvolution') require('cudnn.RNN') +require('cudnn.BLSTM') require('cudnn.LSTM') require('cudnn.GRU') -require('cudnn.BLSTM') require('cudnn.functional') require('cudnn.convert') From 589fdfd5552c0e9a7fe7b4192fc24de280b3bfc2 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 18:48:15 +0100 Subject: [PATCH 33/49] Fixed module names --- GRU.lua | 2 +- LSTM.lua | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GRU.lua b/GRU.lua index 73c6ed1..2f5147b 100644 --- a/GRU.lua +++ b/GRU.lua @@ -1,6 +1,6 @@ local GRU, parent = torch.class('cudnn.GRU', 'cudnn.RNN') -function BLSTM:__init(inputSize, hiddenSize, numLayers) +function GRU:__init(inputSize, hiddenSize, numLayers) parent.__init(self,inputSize, hiddenSize, numLayers) self.mode = 'CUDNN_GRU' self:reset() diff --git a/LSTM.lua b/LSTM.lua index 096fd6b..6bb4c63 100644 --- a/LSTM.lua +++ b/LSTM.lua @@ -1,6 +1,6 @@ -local BLSTM, parent = torch.class('cudnn.LSTM', 'cudnn.RNN') +local LSTM, parent = torch.class('cudnn.LSTM', 'cudnn.RNN') -function BLSTM:__init(inputSize, hiddenSize, numLayers) +function LSTM:__init(inputSize, hiddenSize, numLayers) parent.__init(self,inputSize, hiddenSize, numLayers) self.mode = 'CUDNN_LSTM' self:reset() From 04a1ca962ad99d210fe5f52fe1b72fcefc339ffb Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:06:12 +0100 Subject: [PATCH 34/49] added batchfirst param --- RNN.lua | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/RNN.lua b/RNN.lua index 400c978..45f38e4 100644 --- a/RNN.lua +++ b/RNN.lua @@ -2,7 +2,7 @@ local RNN, parent = torch.class('cudnn.RNN', 'nn.Module') local ffi = require 'ffi' local errcheck = cudnn.errcheck -function RNN:__init(inputSize, hiddenSize, numLayers) +function RNN:__init(inputSize, hiddenSize, numLayers, batchFirst) parent.__init(self) self.datatype = 'CUDNN_DATA_FLOAT' @@ -17,6 +17,7 @@ function RNN:__init(inputSize, hiddenSize, numLayers) self.mode = 'CUDNN_RNN_RELU' self.dropout = 0 self.seed = 0x01234567 + self.batchFirst = batchFirst or false self.gradInput = torch.CudaTensor() self.output = torch.CudaTensor() @@ -235,7 +236,9 @@ end function RNN:updateOutput(input) assert(input:dim() == 3, 'input must have 3 dimensions: seqLength, miniBatch, inputSize') - + if(self.batchFirst) then + input = input:transpose(1, 2) + end -- Decide which descriptors/tensors need to be updated. local resetRNN = not self.dropoutDesc or not self.rnnDesc local resetIO = not self.xDescs or not self.yDescs @@ -357,6 +360,10 @@ function RNN:updateGradInput(input, gradOutput) assert(input:size(2) == self.miniBatch, 'input has incorrect minibatch size!') assert(input:size(3) == self.inputSize, 'input has incorrect size!') + if(self.batchFirst) then + gradOutput = gradOutput:transpose(1, 2) + end + assert(gradOutput:isSameSizeAs(self.output), 'gradOutput has incorrect size!') assert(self.train, 'updateGradInput can only be called when training!') From e7ce48b97c7ba66f8f89e715a3095110e2ac43a9 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:11:40 +0100 Subject: [PATCH 35/49] Put tranpose at top of method call --- RNN.lua | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/RNN.lua b/RNN.lua index 45f38e4..0ef555c 100644 --- a/RNN.lua +++ b/RNN.lua @@ -235,10 +235,10 @@ function RNN:makeContiguous(input, gradOutput) end function RNN:updateOutput(input) + if (self.batchFirst) then + input = input:transpose(1, 2) + end assert(input:dim() == 3, 'input must have 3 dimensions: seqLength, miniBatch, inputSize') - if(self.batchFirst) then - input = input:transpose(1, 2) - end -- Decide which descriptors/tensors need to be updated. local resetRNN = not self.dropoutDesc or not self.rnnDesc local resetIO = not self.xDescs or not self.yDescs @@ -355,15 +355,14 @@ function RNN:updateOutput(input) end function RNN:updateGradInput(input, gradOutput) + if (self.batchFirst) then + gradOutput = gradOutput:transpose(1, 2) + end assert(input:dim() == 3, 'input should have 3 dimensions: seqLength, miniBatch, inputSize') assert(input:size(1) == self.seqLength, 'input has incorrect sequence length!') assert(input:size(2) == self.miniBatch, 'input has incorrect minibatch size!') assert(input:size(3) == self.inputSize, 'input has incorrect size!') - if(self.batchFirst) then - gradOutput = gradOutput:transpose(1, 2) - end - assert(gradOutput:isSameSizeAs(self.output), 'gradOutput has incorrect size!') assert(self.train, 'updateGradInput can only be called when training!') From a6ab5069b8fa26fee0c5fa32e79438035d4c8da3 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:14:48 +0100 Subject: [PATCH 36/49] Added batchFirst to accGrad --- RNN.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index 0ef555c..407f85d 100644 --- a/RNN.lua +++ b/RNN.lua @@ -356,7 +356,8 @@ end function RNN:updateGradInput(input, gradOutput) if (self.batchFirst) then - gradOutput = gradOutput:transpose(1, 2) + input = input:transpose(1, 2) + gradOutput = gradOutput:tranpose(1, 2) end assert(input:dim() == 3, 'input should have 3 dimensions: seqLength, miniBatch, inputSize') assert(input:size(1) == self.seqLength, 'input has incorrect sequence length!') @@ -431,6 +432,10 @@ function RNN:updateGradInput(input, gradOutput) end function RNN:accGradParameters(input, gradOutput, scale) + if (self.batchFirst) then + input = input:transpose(1, 2) + gradOutput = gradOutput:tranpose(1, 2) + end scale = scale or 1 if scale == 0 then return end From f40008bc384f330fa11b554a52faeed5c2042469 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:15:16 +0100 Subject: [PATCH 37/49] Fixed transpose name --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index 407f85d..f8d9d0f 100644 --- a/RNN.lua +++ b/RNN.lua @@ -434,7 +434,7 @@ end function RNN:accGradParameters(input, gradOutput, scale) if (self.batchFirst) then input = input:transpose(1, 2) - gradOutput = gradOutput:tranpose(1, 2) + gradOutput = gradOutput:transpose(1, 2) end scale = scale or 1 if scale == 0 then return end From ba105f0287080122a4fcf1bd487cff2f3525dc26 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:15:52 +0100 Subject: [PATCH 38/49] Fixed transpose name for grads --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index f8d9d0f..af4b8b3 100644 --- a/RNN.lua +++ b/RNN.lua @@ -357,7 +357,7 @@ end function RNN:updateGradInput(input, gradOutput) if (self.batchFirst) then input = input:transpose(1, 2) - gradOutput = gradOutput:tranpose(1, 2) + gradOutput = gradOutput:transpose(1, 2) end assert(input:dim() == 3, 'input should have 3 dimensions: seqLength, miniBatch, inputSize') assert(input:size(1) == self.seqLength, 'input has incorrect sequence length!') From d9062eeb567d886cc2b39f884ef59206a6fd48c5 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:43:45 +0100 Subject: [PATCH 39/49] Added all transpose operations --- RNN.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index af4b8b3..cf2bf52 100644 --- a/RNN.lua +++ b/RNN.lua @@ -350,7 +350,9 @@ function RNN:updateOutput(input) self.cyDesc[0], cy:data(), self.workspace:data(), self.workspace:size(1) * 4) -- sizeof(float) end - + if (self.batchFirst) then + self.output = self.output:transpose(1, 2) + end return self.output end @@ -358,6 +360,7 @@ function RNN:updateGradInput(input, gradOutput) if (self.batchFirst) then input = input:transpose(1, 2) gradOutput = gradOutput:transpose(1, 2) + self.output = self.output:transpose(1, 2) end assert(input:dim() == 3, 'input should have 3 dimensions: seqLength, miniBatch, inputSize') assert(input:size(1) == self.seqLength, 'input has incorrect sequence length!') @@ -428,6 +431,9 @@ function RNN:updateGradInput(input, gradOutput) self.cxDesc[0], dcx:data(), self.workspace:data(), self.workspace:size(1) * 4, -- sizeof(float) self.reserve:data(), self.reserve:size(1) * 4) -- sizeof(float) + if (self.batchFirst) then + self.gradInput = self.gradInput:transpose(1, 2) + end return self.gradInput end From df632a972c5ab938d183a34f949ec2182562c0cb Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:49:11 +0100 Subject: [PATCH 40/49] Added batchFirst to test params --- test/test_rnn.lua | 71 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 82a72ee..864dcf7 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -31,6 +31,25 @@ function cudnntest.testRNNRELU() mytester:assertalmosteq(checkSums.localSumdw, 1.453750E+09, tolerance, 'checkSum with reference for localSumdw failed') end +function cudnntest.testRNNBatchFirst() + local miniBatch = 64 + local seqLength = 20 + local hiddenSize = 512 + local numberOfLayers = 2 + local numberOfLinearLayers = 2 + local batchFirst = true + local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers, batchFirst) + rnn.mode = 'CUDNN_RNN_RELU' + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst) + + -- Checksums to check against are retrieved from cudnn RNN sample. + mytester:assertalmosteq(checkSums.localSumi, 1.315793E+06, tolerance, 'checkSum with reference for localsumi failed') + mytester:assertalmosteq(checkSums.localSumh, 1.315212E+05, tolerance, 'checkSum with reference for localSumh failed') + mytester:assertalmosteq(checkSums.localSumdi, 6.676003E+01, tolerance, 'checkSum with reference for localSumdi failed') + mytester:assertalmosteq(checkSums.localSumdh, 6.425067E+01, tolerance, 'checkSum with reference for localSumdh failed') + mytester:assertalmosteq(checkSums.localSumdw, 1.453750E+09, tolerance, 'checkSum with reference for localSumdw failed') +end + function cudnntest.testRNNTANH() local miniBatch = 64 local seqLength = 20 @@ -91,12 +110,13 @@ function cudnntest.testBiDirectionalRELURNN() local numberOfLayers = 2 local numberOfLinearLayers = 2 local nbDirections = 2 + local batchFirst = false local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_RNN_RELU' rnn.numDirections = 2 - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.388634E+01, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 1.288997E+01, tolerance, 'checkSum with reference for localSumh failed') @@ -112,12 +132,13 @@ function cudnntest.testBiDirectionalTANHRNN() local numberOfLayers = 2 local numberOfLinearLayers = 2 local nbDirections = 2 + local batchFirst = false local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_RNN_TANH' rnn.numDirections = 2 - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 1.388634E+01, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 1.288997E+01, tolerance, 'checkSum with reference for localSumh failed') @@ -133,9 +154,10 @@ function cudnntest.testBiDirectionalLSTMRNN() local numberOfLayers = 2 local numberOfLinearLayers = 8 local nbDirections = 2 + local batchFirst = false local rnn = cudnn.BLSTM(hiddenSize, hiddenSize, numberOfLayers) - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 3.134097E+04, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumc, 3.845626E+00, tolerance, 'checkSum with reference for localSumc failed') @@ -153,12 +175,13 @@ function cudnntest.testBiDirectionalGRURNN() local numberOfLayers = 2 local numberOfLinearLayers = 6 local nbDirections = 2 + local batchFirst = false local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) rnn.bidirectional = 'CUDNN_BIDIRECTIONAL' rnn.mode = 'CUDNN_GRU' rnn.numDirections = 2 - local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, nbDirections) + local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst, nbDirections) -- Checksums to check against are retrieved from cudnn RNN sample. mytester:assertalmosteq(checkSums.localSumi, 6.555183E+04, tolerance, 'checkSum with reference for localsumi failed') mytester:assertalmosteq(checkSums.localSumh, 5.830924E+00, tolerance, 'checkSum with reference for localSumh failed') @@ -170,14 +193,18 @@ end --[[ -- Method gets Checksums of RNN to compare with ref Checksums in cudnn RNN C sample. -- ]] -function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, biDirectional) - local biDirectionalScale = biDirectional or 1 +function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst, nbDirections) + local biDirectionalScale = nbDirections or 1 -- Reset the rnn and weight descriptor (since we are manually setting values for matrix/bias. rnn:reset() rnn:resetWeightDescriptor() - local input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. - - if (biDirectional) then + local input + if (batchFirst) then + input = torch.CudaTensor(miniBatch, seqLength, hiddenSize):fill(1) + else + input = torch.CudaTensor(seqLength, miniBatch, hiddenSize):fill(1) -- Input initialised to 1s. + end + if (biDirectionalScale == 2) then rnn.weight:fill(1 / rnn.weight:size(1)) else -- Matrices are initialised to 1 / matrixSize, biases to 1. @@ -253,15 +280,31 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe rnn.gradCellOutput = torch.CudaTensor(numberOfLayers * biDirectionalScale, miniBatch, hiddenSize):fill(1) local testOutputi = rnn:forward(input) -- gradInput set to 1s. - local gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * biDirectionalScale):fill(1) + local gradInput + if(batchFirst) then + gradInput = torch.CudaTensor(miniBatch, seqLength, hiddenSize * biDirectionalScale):fill(1) + else + gradInput = torch.CudaTensor(seqLength, miniBatch, hiddenSize * biDirectionalScale):fill(1) + end rnn:backward(input, gradInput) -- Sum up all values for each. local localSumi = 0 local localSumh = 0 local localSumc = 0 - for m = 1, seqLength do - for j = 1, miniBatch do + + local mLength + local jLength + if(batchFirst) then + mLength = miniBatch + jLength = seqLength + else + mLength = seqLength + jLength = miniBatch + end + + for m = 1, mLength do + for j = 1, jLength do for i = 1, hiddenSize * biDirectionalScale do localSumi = localSumi + testOutputi[m][j][i] end @@ -280,8 +323,8 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe local localSumdi = 0 local localSumdh = 0 local localSumdc = 0 - for m = 1, seqLength do - for j = 1, miniBatch do + for m = 1, mLength do + for j = 1, jLength do for i = 1, hiddenSize do localSumdi = localSumdi + rnn.gradInput[m][j][i] end From e2102080e7f663c6b9f3b95ac601fc2411979952 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Fri, 15 Apr 2016 23:51:21 +0100 Subject: [PATCH 41/49] Added small comment to clarify batchFirst --- RNN.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RNN.lua b/RNN.lua index cf2bf52..f5d9441 100644 --- a/RNN.lua +++ b/RNN.lua @@ -17,7 +17,7 @@ function RNN:__init(inputSize, hiddenSize, numLayers, batchFirst) self.mode = 'CUDNN_RNN_RELU' self.dropout = 0 self.seed = 0x01234567 - self.batchFirst = batchFirst or false + self.batchFirst = batchFirst or false -- Set to true for batch x time x inputdim. self.gradInput = torch.CudaTensor() self.output = torch.CudaTensor() From c420df1ff655b3364d9081dd02cc130b7f3deaa2 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 00:10:28 +0100 Subject: [PATCH 42/49] Added description of recurrent modules --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 78efddd..9de7f4a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,14 @@ cudnn.SpatialCrossEntropyCriterion() -- A spatial version of LogSoftMax + cudnn.VolumetricConvolution(nInputPlane, nOutputPlane, kT, kW, kH, dT, dW, dH, padT, padW, padH) cudnn.VolumetricMaxPooling(kT, kW, kH, dT, dW, dH, padT, padW, padH) cudnn.VolumetricAveragePooling(kT, kW, kH, dT, dW, dH, padT, padW, padH) + +-- Recurrent Modules + +-- All inputs have to be 3D. Accepts input of seqLength x batch x inputDim, or batch x seqLength x inputDim if batchFirst set to true. +cudnn.RNN(inputDim, outputDim, numberOfLayers, [batchFirst = false]) +cudnn.LSTM(inputDim, outputDim, numberOfLayers, [batchFirst = false]) +cudnn.GRU(inputDim, outputDim, numberOfLayers, [batchFirst = false]) +cudnn.BLSTM(inputDim, outputDim, numberOfLayers, [batchFirst = false]) ``` ### Modes From a334ae2cedced0103340b4595726f87d995adc13 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 09:51:08 +0100 Subject: [PATCH 43/49] Added separate RNN modules --- RNNReLU.lua | 6 ++++++ RNNTanh.lua | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 RNNReLU.lua create mode 100644 RNNTanh.lua diff --git a/RNNReLU.lua b/RNNReLU.lua new file mode 100644 index 0000000..39ebdd2 --- /dev/null +++ b/RNNReLU.lua @@ -0,0 +1,6 @@ +local RNNReLU, parent = torch.class('cudnn.RNNReLU', 'cudnn.RNN') + +function RNNReLU:__init(inputSize, hiddenSize, numLayers) + parent.__init(self,inputSize, hiddenSize, numLayers) + self.mode = 'CUDNN_RNN_RELU' +end diff --git a/RNNTanh.lua b/RNNTanh.lua new file mode 100644 index 0000000..be91a09 --- /dev/null +++ b/RNNTanh.lua @@ -0,0 +1,6 @@ +local RNNTanh, parent = torch.class('cudnn.RNNTanh', 'cudnn.RNN') + +function RNNTanh:__init(inputSize, hiddenSize, numLayers) + parent.__init(self,inputSize, hiddenSize, numLayers) + self.mode = 'CUDNN_RNN_TANH' +end From 079a11c977d1fa71f9839f981d81be50631a6905 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 09:52:28 +0100 Subject: [PATCH 44/49] Added modules to init --- init.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init.lua b/init.lua index 9ff687a..318570b 100644 --- a/init.lua +++ b/init.lua @@ -123,6 +123,8 @@ require('cudnn.VolumetricBatchNormalization') require('cudnn.SpatialCrossEntropyCriterion') require('cudnn.TemporalConvolution') require('cudnn.RNN') +require('cudnn.RNNTanh') +require('cudnn.RNNReLU') require('cudnn.BLSTM') require('cudnn.LSTM') require('cudnn.GRU') From 0a9f8ca9b139e1ccd6c599b256d67a86f83547ad Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 10:21:34 +0100 Subject: [PATCH 45/49] Added RNNReLU/Tanh description --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9de7f4a..b12e2fa 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,8 @@ cudnn.VolumetricAveragePooling(kT, kW, kH, dT, dW, dH, padT, padW, padH) -- Recurrent Modules -- All inputs have to be 3D. Accepts input of seqLength x batch x inputDim, or batch x seqLength x inputDim if batchFirst set to true. -cudnn.RNN(inputDim, outputDim, numberOfLayers, [batchFirst = false]) +cudnn.RNNReLU(inputDim, outputDim, numberOfLayers, [batchFirst = false]) +cudnn.RNNTanh(inputDim, outputDim, numberOfLayers, [batchFirst = false]) cudnn.LSTM(inputDim, outputDim, numberOfLayers, [batchFirst = false]) cudnn.GRU(inputDim, outputDim, numberOfLayers, [batchFirst = false]) cudnn.BLSTM(inputDim, outputDim, numberOfLayers, [batchFirst = false]) From 2cff977d9937ec972451a4e68bd1f403a9a98517 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 13:10:59 +0100 Subject: [PATCH 46/49] Added batchFirst to params --- BLSTM.lua | 4 ++-- RNNReLU.lua | 4 ++-- RNNTanh.lua | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/BLSTM.lua b/BLSTM.lua index e452cb6..8feebf1 100644 --- a/BLSTM.lua +++ b/BLSTM.lua @@ -1,7 +1,7 @@ local BLSTM, parent = torch.class('cudnn.BLSTM', 'cudnn.RNN') -function BLSTM:__init(inputSize, hiddenSize, numLayers) - parent.__init(self,inputSize, hiddenSize, numLayers) +function BLSTM:__init(inputSize, hiddenSize, numLayers, batchFirst) + parent.__init(self,inputSize, hiddenSize, numLayers, batchFirst) self.bidirectional = 'CUDNN_BIDIRECTIONAL' self.mode = 'CUDNN_LSTM' self.numDirections = 2 diff --git a/RNNReLU.lua b/RNNReLU.lua index 39ebdd2..d17ff9c 100644 --- a/RNNReLU.lua +++ b/RNNReLU.lua @@ -1,6 +1,6 @@ local RNNReLU, parent = torch.class('cudnn.RNNReLU', 'cudnn.RNN') -function RNNReLU:__init(inputSize, hiddenSize, numLayers) - parent.__init(self,inputSize, hiddenSize, numLayers) +function RNNReLU:__init(inputSize, hiddenSize, numLayers, batchFirst) + parent.__init(self,inputSize, hiddenSize, numLayers, batchFirst) self.mode = 'CUDNN_RNN_RELU' end diff --git a/RNNTanh.lua b/RNNTanh.lua index be91a09..912a83e 100644 --- a/RNNTanh.lua +++ b/RNNTanh.lua @@ -1,6 +1,6 @@ local RNNTanh, parent = torch.class('cudnn.RNNTanh', 'cudnn.RNN') -function RNNTanh:__init(inputSize, hiddenSize, numLayers) - parent.__init(self,inputSize, hiddenSize, numLayers) +function RNNTanh:__init(inputSize, hiddenSize, numLayers, batchFirst) + parent.__init(self,inputSize, hiddenSize, numLayers, batchFirst) self.mode = 'CUDNN_RNN_TANH' end From fc7e76116a60065073331076638ac317ee384c1c Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 13:12:40 +0100 Subject: [PATCH 47/49] Changed module calls in tests --- test/test_rnn.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 864dcf7..21cc876 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -19,7 +19,7 @@ function cudnntest.testRNNRELU() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 2 - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNNReLU(hiddenSize, hiddenSize, numberOfLayers) rnn.mode = 'CUDNN_RNN_RELU' local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) @@ -38,7 +38,7 @@ function cudnntest.testRNNBatchFirst() local numberOfLayers = 2 local numberOfLinearLayers = 2 local batchFirst = true - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers, batchFirst) + local rnn = cudnn.RNNReLU(hiddenSize, hiddenSize, numberOfLayers, batchFirst) rnn.mode = 'CUDNN_RNN_RELU' local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn, batchFirst) @@ -56,7 +56,7 @@ function cudnntest.testRNNTANH() local hiddenSize = 512 local numberOfLayers = 2 local numberOfLinearLayers = 2 - local rnn = cudnn.RNN(hiddenSize, hiddenSize, numberOfLayers) + local rnn = cudnn.RNNTanh(hiddenSize, hiddenSize, numberOfLayers) rnn.mode = 'CUDNN_RNN_TANH' local checkSums = getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numberOfLinearLayers, rnn) From 50d4f7cc2cebc4b1982a6eda702deb07e057a58c Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Sat, 16 Apr 2016 13:28:01 +0100 Subject: [PATCH 48/49] Use torch sum instead of manual loop --- test/test_rnn.lua | 59 ++++++----------------------------------------- 1 file changed, 7 insertions(+), 52 deletions(-) diff --git a/test/test_rnn.lua b/test/test_rnn.lua index 21cc876..e7ee3de 100644 --- a/test/test_rnn.lua +++ b/test/test_rnn.lua @@ -289,60 +289,15 @@ function getRNNCheckSums(miniBatch, seqLength, hiddenSize, numberOfLayers, numbe rnn:backward(input, gradInput) -- Sum up all values for each. - local localSumi = 0 - local localSumh = 0 - local localSumc = 0 + local localSumi = torch.sum(testOutputi) + local localSumh = torch.sum(rnn.hiddenOutput) + local localSumc = torch.sum(rnn.cellOutput) - local mLength - local jLength - if(batchFirst) then - mLength = miniBatch - jLength = seqLength - else - mLength = seqLength - jLength = miniBatch - end - - for m = 1, mLength do - for j = 1, jLength do - for i = 1, hiddenSize * biDirectionalScale do - localSumi = localSumi + testOutputi[m][j][i] - end - end - end + local localSumdi = torch.sum(rnn.gradInput) + local localSumdh = torch.sum(rnn.gradHiddenInput) + local localSumdc = torch.sum(rnn.gradCellInput) - for m = 1, numberOfLayers * biDirectionalScale do - for j = 1, miniBatch do - for i = 1, hiddenSize do - localSumh = localSumh + rnn.hiddenOutput[m][j][i] - localSumc = localSumc + rnn.cellOutput[m][j][i] - end - end - end - - local localSumdi = 0 - local localSumdh = 0 - local localSumdc = 0 - for m = 1, mLength do - for j = 1, jLength do - for i = 1, hiddenSize do - localSumdi = localSumdi + rnn.gradInput[m][j][i] - end - end - end - for m = 1, numberOfLayers * biDirectionalScale do - for j = 1, miniBatch do - for i = 1, hiddenSize do - localSumdh = localSumdh + rnn.gradHiddenInput[m][j][i] - localSumdc = localSumdc + rnn.gradCellInput[m][j][i] - end - end - end - - local localSumdw = 0 - for m = 1, rnn.gradWeight:size(1) do - localSumdw = localSumdw + rnn.gradWeight[m] - end + local localSumdw = torch.sum(rnn.gradWeight) local checkSums = { localSumi = localSumi, From b89f6c232d9c1e59793db759f009958bbf2c7f0e Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 18 Apr 2016 18:41:02 +0100 Subject: [PATCH 49/49] Put resizing of tensors in a better place --- RNN.lua | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/RNN.lua b/RNN.lua index f5d9441..a4840f7 100644 --- a/RNN.lua +++ b/RNN.lua @@ -33,14 +33,6 @@ function RNN:__init(inputSize, hiddenSize, numLayers, batchFirst) self:reset() end -function RNN:resizeOutput(tensor) - return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize * self.numDirections) -end - -function RNN:resizeHidden(tensor) - return tensor:resize(self.numLayers * self.numDirections, self.miniBatch, self.hiddenSize) -end - function RNN:reset(stdv) stdv = stdv or 1.0 / math.sqrt(self.hiddenSize) @@ -234,6 +226,14 @@ function RNN:makeContiguous(input, gradOutput) return input, gradOutput end +function RNN:resizeOutput(tensor) + return tensor:resize(self.seqLength, self.miniBatch, self.hiddenSize * self.numDirections) +end + +function RNN:resizeHidden(tensor) + return tensor:resize(self.numLayers * self.numDirections, self.miniBatch, self.hiddenSize) +end + function RNN:updateOutput(input) if (self.batchFirst) then input = input:transpose(1, 2)