From 8c9f88d123d0501fbbd44e7f41e9240ee7173f49 Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Fri, 17 May 2019 09:44:36 -0700 Subject: [PATCH 1/6] Add all VJP functions, need to write tests. --- stdlib/public/TensorFlow/Gradients.swift | 64 ++++++++++++++++++++++++ stdlib/public/TensorFlow/Ops.swift | 15 ++++++ 2 files changed, 79 insertions(+) diff --git a/stdlib/public/TensorFlow/Gradients.swift b/stdlib/public/TensorFlow/Gradients.swift index 7a268c638ecab..c5fc279abda69 100644 --- a/stdlib/public/TensorFlow/Gradients.swift +++ b/stdlib/public/TensorFlow/Gradients.swift @@ -635,3 +635,67 @@ func _vjpRelu( ) -> (Tensor, (Tensor) -> Tensor) { return (relu(x), { v in Tensor(x .> 0) * v }) } + +//===----------------------------------------------------------------------===// +// Broadcasting +//===----------------------------------------------------------------------===// + +extension Tensor where Scalar : TensorFlowFloatingPoint { + @inlinable + func _vjpBroadcast( + toShape shape: Tensor + ) -> (Tensor, (Tensor) -> Tensor) { + return (broadcast(toShape: shape), { [origShape = self.shapeTensor] v in + v.unbroadcast(toShape: origShape) + }) + } + + @inlinable + func _vjpBroadcast( + to shape: TensorShape + ) -> (Tensor, (Tensor) -> Tensor) { + return (broadcast(to: shape), { [origShape = self.shape] v in + v.unbroadcast(to: origShape) + }) + } + + @inlinable + func _vjpBroadcast( + like other: Tensor + ) -> (Tensor, (Tensor) -> Tensor) + where OtherScalar : TensorFlowScalar { + return (broadcast(like: other), { [origShape = self.shapeTensor] v in + v.unbroadcast(like: origShape) + }) + } +} + +extension Tensor where Scalar : Numeric { + @inlinable + func _vjpUnbroadcast( + toShape shape: Tensor + ) -> (Tensor, (Tensor) -> Tensor) { + return (unbroadcast(toShape: shape), { [origShape = self.shapeTensor] v in + v.broadcast(toShape: origShape) + }) + } + + @inlinable + func _vjpUnbroadcast( + to shape: TensorShape + ) -> (Tensor, (Tensor) -> Tensor) { + return (unbroadcast(to: shape), { [origShape = self.shape] v in + v.broadcast(to: origShape) + }) + } + + @inlinable + func _vjpUnbroadcast( + like other: Tensor + ) -> (Tensor, (Tensor) -> Tensor) + where OtherScalar : TensorFlowScalar { + return (unbroadcast(like: other), { [origShape = self.shapeTensor] v in + v.broadcast(like: origShape) + }) + } +} diff --git a/stdlib/public/TensorFlow/Ops.swift b/stdlib/public/TensorFlow/Ops.swift index 78b0b3b6d1846..80d6327823f02 100644 --- a/stdlib/public/TensorFlow/Ops.swift +++ b/stdlib/public/TensorFlow/Ops.swift @@ -1600,11 +1600,17 @@ public extension Tensor { public extension Tensor { @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpBroadcast(toShape:) + where Scalar : TensorFlowFloatingPoint) func broadcast(toShape shape: Tensor) -> Tensor { return Raw.broadcastTo(self, shape: shape) } @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpBroadcast(to:) + where Scalar : TensorFlowFloatingPoint) func broadcast(to shape: TensorShape) -> Tensor { return broadcast(toShape: Tensor(shape.dimensions.map(Int32.init))) } @@ -1612,6 +1618,9 @@ public extension Tensor { /// Broadcast to the same shape as the specified `Tensor`. /// - Precondition: The specified shape must be compatible for broadcasting. @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpBroadcast(like:) + where Scalar : TensorFlowFloatingPoint) func broadcast(like other: Tensor) -> Tensor { return broadcast(toShape: other.shapeTensor) } @@ -1619,6 +1628,8 @@ public extension Tensor { public extension Tensor where Scalar : Numeric { @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpUnbroadcast(toShape:)) func unbroadcast(toShape otherShape: Tensor) -> Tensor { let rankDiff = (rankTensor - otherShape.scalarCountTensor).rankLifted() let ones: Tensor = Raw.fill(dims: rankDiff, value: Tensor(1)) @@ -1631,11 +1642,15 @@ public extension Tensor where Scalar : Numeric { } @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpUnbroadcast(like:)) func unbroadcast(like other: Tensor) -> Tensor { return unbroadcast(toShape: other.shapeTensor) } @inlinable + // SWIFT_ENABLE_TENSORFLOW + @differentiable(wrt: self, vjp: _vjpUnbroadcast(to:)) func unbroadcast(to shape: TensorShape) -> Tensor { return unbroadcast(toShape: Tensor(shape.dimensions.map(Int32.init))) } From af4859a4f2e52111e21ff70a890a0a3fbb6e7c09 Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Fri, 17 May 2019 17:06:20 -0700 Subject: [PATCH 2/6] PR feedback batch #1. --- stdlib/public/TensorFlow/Gradients.swift | 22 ---------------------- stdlib/public/TensorFlow/Ops.swift | 16 ++++++---------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/stdlib/public/TensorFlow/Gradients.swift b/stdlib/public/TensorFlow/Gradients.swift index c5fc279abda69..605fdd68afa5d 100644 --- a/stdlib/public/TensorFlow/Gradients.swift +++ b/stdlib/public/TensorFlow/Gradients.swift @@ -658,19 +658,7 @@ extension Tensor where Scalar : TensorFlowFloatingPoint { v.unbroadcast(to: origShape) }) } - - @inlinable - func _vjpBroadcast( - like other: Tensor - ) -> (Tensor, (Tensor) -> Tensor) - where OtherScalar : TensorFlowScalar { - return (broadcast(like: other), { [origShape = self.shapeTensor] v in - v.unbroadcast(like: origShape) - }) - } -} -extension Tensor where Scalar : Numeric { @inlinable func _vjpUnbroadcast( toShape shape: Tensor @@ -688,14 +676,4 @@ extension Tensor where Scalar : Numeric { v.broadcast(to: origShape) }) } - - @inlinable - func _vjpUnbroadcast( - like other: Tensor - ) -> (Tensor, (Tensor) -> Tensor) - where OtherScalar : TensorFlowScalar { - return (unbroadcast(like: other), { [origShape = self.shapeTensor] v in - v.broadcast(like: origShape) - }) - } } diff --git a/stdlib/public/TensorFlow/Ops.swift b/stdlib/public/TensorFlow/Ops.swift index 80d6327823f02..4aa12c7f74ceb 100644 --- a/stdlib/public/TensorFlow/Ops.swift +++ b/stdlib/public/TensorFlow/Ops.swift @@ -1600,7 +1600,6 @@ public extension Tensor { public extension Tensor { @inlinable - // SWIFT_ENABLE_TENSORFLOW @differentiable(wrt: self, vjp: _vjpBroadcast(toShape:) where Scalar : TensorFlowFloatingPoint) func broadcast(toShape shape: Tensor) -> Tensor { @@ -1608,7 +1607,6 @@ public extension Tensor { } @inlinable - // SWIFT_ENABLE_TENSORFLOW @differentiable(wrt: self, vjp: _vjpBroadcast(to:) where Scalar : TensorFlowFloatingPoint) func broadcast(to shape: TensorShape) -> Tensor { @@ -1618,8 +1616,7 @@ public extension Tensor { /// Broadcast to the same shape as the specified `Tensor`. /// - Precondition: The specified shape must be compatible for broadcasting. @inlinable - // SWIFT_ENABLE_TENSORFLOW - @differentiable(wrt: self, vjp: _vjpBroadcast(like:) + @differentiable(wrt: self where Scalar : TensorFlowFloatingPoint) func broadcast(like other: Tensor) -> Tensor { return broadcast(toShape: other.shapeTensor) @@ -1628,8 +1625,8 @@ public extension Tensor { public extension Tensor where Scalar : Numeric { @inlinable - // SWIFT_ENABLE_TENSORFLOW - @differentiable(wrt: self, vjp: _vjpUnbroadcast(toShape:)) + @differentiable(wrt: self, vjp: _vjpUnbroadcast(toShape:) + where Scalar : TensorFlowFloatingPoint) func unbroadcast(toShape otherShape: Tensor) -> Tensor { let rankDiff = (rankTensor - otherShape.scalarCountTensor).rankLifted() let ones: Tensor = Raw.fill(dims: rankDiff, value: Tensor(1)) @@ -1642,15 +1639,14 @@ public extension Tensor where Scalar : Numeric { } @inlinable - // SWIFT_ENABLE_TENSORFLOW - @differentiable(wrt: self, vjp: _vjpUnbroadcast(like:)) + @differentiable(wrt: self where Scalar : TensorFlowFloatingPoint) func unbroadcast(like other: Tensor) -> Tensor { return unbroadcast(toShape: other.shapeTensor) } @inlinable - // SWIFT_ENABLE_TENSORFLOW - @differentiable(wrt: self, vjp: _vjpUnbroadcast(to:)) + @differentiable(wrt: self, vjp: _vjpUnbroadcast(to:) + where Scalar : TensorFlowFloatingPoint) func unbroadcast(to shape: TensorShape) -> Tensor { return unbroadcast(toShape: Tensor(shape.dimensions.map(Int32.init))) } From 105089bd3d6d916e53e568a18c486c9be68f6e64 Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Fri, 17 May 2019 18:23:41 -0700 Subject: [PATCH 3/6] Use closure call to remove VJPs --- stdlib/public/TensorFlow/Gradients.swift | 18 ------------------ stdlib/public/TensorFlow/Ops.swift | 10 ++++------ 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/stdlib/public/TensorFlow/Gradients.swift b/stdlib/public/TensorFlow/Gradients.swift index 605fdd68afa5d..d6a9c2ef39654 100644 --- a/stdlib/public/TensorFlow/Gradients.swift +++ b/stdlib/public/TensorFlow/Gradients.swift @@ -649,15 +649,6 @@ extension Tensor where Scalar : TensorFlowFloatingPoint { v.unbroadcast(toShape: origShape) }) } - - @inlinable - func _vjpBroadcast( - to shape: TensorShape - ) -> (Tensor, (Tensor) -> Tensor) { - return (broadcast(to: shape), { [origShape = self.shape] v in - v.unbroadcast(to: origShape) - }) - } @inlinable func _vjpUnbroadcast( @@ -667,13 +658,4 @@ extension Tensor where Scalar : TensorFlowFloatingPoint { v.broadcast(toShape: origShape) }) } - - @inlinable - func _vjpUnbroadcast( - to shape: TensorShape - ) -> (Tensor, (Tensor) -> Tensor) { - return (unbroadcast(to: shape), { [origShape = self.shape] v in - v.broadcast(to: origShape) - }) - } } diff --git a/stdlib/public/TensorFlow/Ops.swift b/stdlib/public/TensorFlow/Ops.swift index 4aa12c7f74ceb..f9aab2e5c433e 100644 --- a/stdlib/public/TensorFlow/Ops.swift +++ b/stdlib/public/TensorFlow/Ops.swift @@ -1607,10 +1607,9 @@ public extension Tensor { } @inlinable - @differentiable(wrt: self, vjp: _vjpBroadcast(to:) - where Scalar : TensorFlowFloatingPoint) + @differentiable(wrt: self where Scalar : TensorFlowFloatingPoint) func broadcast(to shape: TensorShape) -> Tensor { - return broadcast(toShape: Tensor(shape.dimensions.map(Int32.init))) + return broadcast(toShape: Tensor({ shape.dimensions.map(Int32.init) }())) } /// Broadcast to the same shape as the specified `Tensor`. @@ -1645,10 +1644,9 @@ public extension Tensor where Scalar : Numeric { } @inlinable - @differentiable(wrt: self, vjp: _vjpUnbroadcast(to:) - where Scalar : TensorFlowFloatingPoint) + @differentiable(wrt: self where Scalar : TensorFlowFloatingPoint) func unbroadcast(to shape: TensorShape) -> Tensor { - return unbroadcast(toShape: Tensor(shape.dimensions.map(Int32.init))) + return unbroadcast(toShape: Tensor({ shape.dimensions.map(Int32.init) }())) } @inlinable From e5eceff93327c7283260be6c63fa4f1bb10c4bcb Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Fri, 17 May 2019 19:28:21 -0700 Subject: [PATCH 4/6] Start adding tests (un)broadcast(toShape:). --- .../tensor_autodiff_runtime.swift | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift index f287489d4f798..17b33d813ca80 100644 --- a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift +++ b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift @@ -262,4 +262,135 @@ TensorADTests.testAllBackends("Side effects") { expectEqual(Tensor(48), gradient(at: Tensor(4), in: bar)) } +TensorADTests.testAllBackends("broadcast(toShape:)") { + func foo(tensor: Tensor, shape: Tensor) -> Tensor { + tensor.broadcast(toShape: shape) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,] -> [3,3] + pb = pullback(at: Tensor([99, 33, 55])) { x in + foo(tensor: x, shape: Tensor([3, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([3, 6, 9]) + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([4, 8, 12]) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([1, 2, 3]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // Test 4: extremely padded shape as tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // [3,1] -> [3x3] + pb = pullback(at: Tensor([[99, 33, 55]])) { x in + foo(tensor: x, shape: Tensor([3, 3])) + } + + // Test 5: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[3, 6, 9]]) + expectEqual(expected, pb(inputTensor)) + + // Test 6: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[4, 8, 12]]) + expectEqual(expected, pb(inputTensor)) + + // Test 7: same shape as tensor we are differentiating at + inputTensor = Tensor([[1, 2, 3]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) + + // Test 8: extremely padded shape of tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) +} + +TensorADTests.testAllBackends("unbroadcast(toShape:") { + func foo(tensor: Tensor, shape: Tensor) -> Tensor { + tensor.unbroadcast(toShape: shape) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,3] -> [1,3] + let atTensor: Tensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + pb = pullback(at: atTensor) { x in + foo(tensor: x, shape: Tensor([1, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([[1, 2, 3]]) + expected = atTensor + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([2]) + expected = Tensor([ + [2, 2, 2], + [2, 2, 2], + [2, 2, 2]] + ) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([ + [8, 1, 3], + [8, 1, 3], + [8, 1, 3]] + ) + expected = inputTensor + expectEqual(expected, pb(inputTensor)) + + // TODO + // Test 4: extremely padded shape as tensor we are differentiating at + // inputTensor = Tensor([ + // [[8, 1, 3]], + // [[8, 1, 3]], + // [[8, 1, 3]]] + // ) + // expected = Tensor([1, 2, 3]) + // expectEqual(expected, pb(inputTensor)) +} + runAllTests() From 74e1129c7285dcfd2582020521e68f28f89adbf8 Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Sat, 18 May 2019 10:57:16 -0700 Subject: [PATCH 5/6] Add tests to (un)broadcast(to/like:). --- .../tensor_autodiff_runtime.swift | 249 +++++++++++++++++- 1 file changed, 241 insertions(+), 8 deletions(-) diff --git a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift index 17b33d813ca80..94213e73b8730 100644 --- a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift +++ b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift @@ -381,16 +381,249 @@ TensorADTests.testAllBackends("unbroadcast(toShape:") { ) expected = inputTensor expectEqual(expected, pb(inputTensor)) +} + +TensorADTests.testAllBackends("broadcast(like:)") { + func foo(tensor: Tensor, other: Tensor) -> Tensor { + tensor.broadcast(like: other) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,] -> [3,3] + pb = pullback(at: Tensor([99, 33, 55])) { x in + foo(tensor: x, other: Tensor([[1, 2, 3], [1, 2, 3], [1, 2, 3]])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([3, 6, 9]) + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([4, 8, 12]) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([1, 2, 3]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // Test 4: extremely padded shape as tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // [3,1] -> [3x3] + pb = pullback(at: Tensor([[99, 33, 55]])) { x in + foo(tensor: x, other: Tensor([[1, 2, 3], [1, 2, 3], [1, 2, 3]])) + } + + // Test 5: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[3, 6, 9]]) + expectEqual(expected, pb(inputTensor)) + + // Test 6: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[4, 8, 12]]) + expectEqual(expected, pb(inputTensor)) + + // Test 7: same shape as tensor we are differentiating at + inputTensor = Tensor([[1, 2, 3]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) + + // Test 8: extremely padded shape of tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) +} + +TensorADTests.testAllBackends("unbroadcast(like:") { + func foo(tensor: Tensor, other: Tensor) -> Tensor { + tensor.unbroadcast(like: other) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,3] -> [1,3] + let atTensor: Tensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + pb = pullback(at: atTensor) { x in + foo(tensor: x, other: Tensor([[1, 2, 3]])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([[1, 2, 3]]) + expected = atTensor + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([2]) + expected = Tensor([ + [2, 2, 2], + [2, 2, 2], + [2, 2, 2]] + ) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([ + [8, 1, 3], + [8, 1, 3], + [8, 1, 3]] + ) + expected = inputTensor + expectEqual(expected, pb(inputTensor)) +} + +// ............................................................................................ +TensorADTests.testAllBackends("broadcast(to:)") { + func foo(tensor: Tensor, shape: TensorShape) -> Tensor { + tensor.broadcast(to: shape) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,] -> [3,3] + pb = pullback(at: Tensor([99, 33, 55])) { x in + foo(tensor: x, shape: TensorShape([3, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([3, 6, 9]) + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([4, 8, 12]) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([1, 2, 3]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) - // TODO // Test 4: extremely padded shape as tensor we are differentiating at - // inputTensor = Tensor([ - // [[8, 1, 3]], - // [[8, 1, 3]], - // [[8, 1, 3]]] - // ) - // expected = Tensor([1, 2, 3]) - // expectEqual(expected, pb(inputTensor)) + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // [3,1] -> [3x3] + pb = pullback(at: Tensor([[99, 33, 55]])) { x in + foo(tensor: x, shape: TensorShape([3, 3])) + } + + // Test 5: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[3, 6, 9]]) + expectEqual(expected, pb(inputTensor)) + + // Test 6: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[4, 8, 12]]) + expectEqual(expected, pb(inputTensor)) + + // Test 7: same shape as tensor we are differentiating at + inputTensor = Tensor([[1, 2, 3]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) + + // Test 8: extremely padded shape of tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) +} + +TensorADTests.testAllBackends("unbroadcast(to:") { + func foo(tensor: Tensor, shape: TensorShape) -> Tensor { + tensor.unbroadcast(to: shape) + } + + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,3] -> [1,3] + let atTensor: Tensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + pb = pullback(at: atTensor) { x in + foo(tensor: x, shape: TensorShape([1, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([[1, 2, 3]]) + expected = atTensor + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([2]) + expected = Tensor([ + [2, 2, 2], + [2, 2, 2], + [2, 2, 2]] + ) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([ + [8, 1, 3], + [8, 1, 3], + [8, 1, 3]] + ) + expected = inputTensor + expectEqual(expected, pb(inputTensor)) } runAllTests() From 68cc56a901a8ca6c6e1853f4cec7eb57c3dc4967 Mon Sep 17 00:00:00 2001 From: Bart Chrzaszcz Date: Sun, 19 May 2019 15:37:53 -0700 Subject: [PATCH 6/6] Move around tests b/w Tensor and AutoDiff. --- test/TensorFlowRuntime/tensor.swift | 119 +++++++ .../tensor_autodiff_runtime.swift | 337 ++---------------- 2 files changed, 156 insertions(+), 300 deletions(-) diff --git a/test/TensorFlowRuntime/tensor.swift b/test/TensorFlowRuntime/tensor.swift index 1997e0a0a701f..a1510a4ff376c 100644 --- a/test/TensorFlowRuntime/tensor.swift +++ b/test/TensorFlowRuntime/tensor.swift @@ -711,6 +711,125 @@ TensorTests.testAllBackends("Unbroadcast2") { z.array) } +TensorTests.testAllBackends("broadcast(toShape:)") { + func foo(tensor: Tensor, shape: Tensor) -> Tensor { + tensor.broadcast(toShape: shape) + } + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,] -> [3,3] + pb = pullback(at: Tensor([99, 33, 55])) { x in + foo(tensor: x, shape: Tensor([3, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([3, 6, 9]) + expectEqual(expected, pb(inputTensor)) + + // Test 2: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([4, 8, 12]) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([1, 2, 3]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // Test 4: extremely padded shape as tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([1, 2, 3]) + expectEqual(expected, pb(inputTensor)) + + // [3,1] -> [3x3] + pb = pullback(at: Tensor([[99, 33, 55]])) { x in + foo(tensor: x, shape: Tensor([3, 3])) + } + + // Test 5: same shape as parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[3, 6, 9]]) + expectEqual(expected, pb(inputTensor)) + + // Test 6: different shape than parameter of pullback + inputTensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + expected = Tensor([[4, 8, 12]]) + expectEqual(expected, pb(inputTensor)) + + // Test 7: same shape as tensor we are differentiating at + inputTensor = Tensor([[1, 2, 3]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) + + // Test 8: extremely padded shape of tensor we are differentiating at + inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) + expected = Tensor([[1, 2, 3]]) + expectEqual(expected, pb(inputTensor)) +} + +TensorADTests.testAllBackends("unbroadcast(toShape:") { + func foo(tensor: Tensor, shape: Tensor) -> Tensor { + tensor.unbroadcast(toShape: shape) + } + var inputTensor: Tensor + var expected: Tensor + var pb: (Tensor) -> Tensor + + // [3,3] -> [1,3] + let atTensor: Tensor = Tensor([ + [1, 2, 3], + [1, 2, 3], + [1, 2, 3]] + ) + pb = pullback(at: atTensor) { x in + foo(tensor: x, shape: Tensor([1, 3])) + } + + // Test 1: same shape as parameter of pullback + inputTensor = Tensor([[1, 2, 3]]) + expected = atTensor + expectEqual(expected, pb(inputTensor)) + // Test 2: different shape than parameter o + f pullback + inputTensor = Tensor([2]) + expected = Tensor([ + [2, 2, 2], + [2, 2, 2], + [2, 2, 2]] + ) + expectEqual(expected, pb(inputTensor)) + + // Test 3: same shape as tensor we are differentiating at + inputTensor = Tensor([ + [8, 1, 3], + [8, 1, 3], + [8, 1, 3]] + ) + expected = inputTensor + expectEqual(expected, pb(inputTensor)) +} + // TODO: Merge all rank/shape getter tests into one when we support code motion // to avoid sends. diff --git a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift index c11071cb7cef8..d1bac71b0f876 100644 --- a/test/TensorFlowRuntime/tensor_autodiff_runtime.swift +++ b/test/TensorFlowRuntime/tensor_autodiff_runtime.swift @@ -266,120 +266,28 @@ TensorADTests.testAllBackends("broadcast(toShape:)") { func foo(tensor: Tensor, shape: Tensor) -> Tensor { tensor.broadcast(toShape: shape) } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,] -> [3,3] - pb = pullback(at: Tensor([99, 33, 55])) { x in + let pb: (Tensor) -> Tensor = pullback(at: Tensor([99, 33, 55])) { x in foo(tensor: x, shape: Tensor([3, 3])) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([3, 6, 9]) - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([ + let inputTensor: Tensor = Tensor([ [1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]] ) - expected = Tensor([4, 8, 12]) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([1, 2, 3]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // Test 4: extremely padded shape as tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // [3,1] -> [3x3] - pb = pullback(at: Tensor([[99, 33, 55]])) { x in - foo(tensor: x, shape: Tensor([3, 3])) - } - - // Test 5: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([[3, 6, 9]]) - expectEqual(expected, pb(inputTensor)) - - // Test 6: different shape than parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([[4, 8, 12]]) - expectEqual(expected, pb(inputTensor)) - - // Test 7: same shape as tensor we are differentiating at - inputTensor = Tensor([[1, 2, 3]]) - expected = Tensor([[1, 2, 3]]) - expectEqual(expected, pb(inputTensor)) - - // Test 8: extremely padded shape of tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([[1, 2, 3]]) + let expected: Tensor = Tensor([[4, 8, 12]]) expectEqual(expected, pb(inputTensor)) } -TensorADTests.testAllBackends("unbroadcast(toShape:") { - func foo(tensor: Tensor, shape: Tensor) -> Tensor { - tensor.unbroadcast(toShape: shape) +TensorADTests.testAllBackends("broadcast(to:") { + func foo(tensor: Tensor, shape: TensorShape) -> Tensor { + tensor.broadcast(to: shape) } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,3] -> [1,3] - let atTensor: Tensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - pb = pullback(at: atTensor) { x in - foo(tensor: x, shape: Tensor([1, 3])) + let pb: (Tensor) -> Tensor = pullback(at: Tensor([99, 33, 55])) { x in + foo(tensor: x, shape: TensorShape([3, 3])) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([[1, 2, 3]]) - expected = atTensor - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([2]) - expected = Tensor([ - [2, 2, 2], - [2, 2, 2], - [2, 2, 2]] - ) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([ - [8, 1, 3], - [8, 1, 3], - [8, 1, 3]] - ) - expected = inputTensor + let inputTensor: Tensor = Tensor([1, 2, 3]) + let expected: Tensor = Tensor([1, 2, 3]) expectEqual(expected, pb(inputTensor)) } @@ -387,241 +295,70 @@ TensorADTests.testAllBackends("broadcast(like:)") { func foo(tensor: Tensor, other: Tensor) -> Tensor { tensor.broadcast(like: other) } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,] -> [3,3] - pb = pullback(at: Tensor([99, 33, 55])) { x in + let pb: (Tensor) -> Tensor = pullback(at: Tensor([99, 33, 55])) { x in foo(tensor: x, other: Tensor([[1, 2, 3], [1, 2, 3], [1, 2, 3]])) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([3, 6, 9]) - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([4, 8, 12]) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([1, 2, 3]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // Test 4: extremely padded shape as tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // [3,1] -> [3x3] - pb = pullback(at: Tensor([[99, 33, 55]])) { x in - foo(tensor: x, other: Tensor([[1, 2, 3], [1, 2, 3], [1, 2, 3]])) - } - - // Test 5: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([[3, 6, 9]]) - expectEqual(expected, pb(inputTensor)) - - // Test 6: different shape than parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([[4, 8, 12]]) - expectEqual(expected, pb(inputTensor)) - - // Test 7: same shape as tensor we are differentiating at - inputTensor = Tensor([[1, 2, 3]]) - expected = Tensor([[1, 2, 3]]) - expectEqual(expected, pb(inputTensor)) - - // Test 8: extremely padded shape of tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([[1, 2, 3]]) + let inputTensor: Tensor = Tensor([[[[[[1, 2, 3]]]]]]) + let expected: Tensor = Tensor([[1, 2, 3]]) expectEqual(expected, pb(inputTensor)) } -TensorADTests.testAllBackends("unbroadcast(like:") { - func foo(tensor: Tensor, other: Tensor) -> Tensor { - tensor.unbroadcast(like: other) +TensorADTests.testAllBackends("unbroadcast(toShape:)") { + func foo(tensor: Tensor, shape: Tensor) -> Tensor { + tensor.unbroadcast(toShape: shape) } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,3] -> [1,3] let atTensor: Tensor = Tensor([ [1, 2, 3], [1, 2, 3], [1, 2, 3]] ) - pb = pullback(at: atTensor) { x in - foo(tensor: x, other: Tensor([[1, 2, 3]])) + let pb: (Tensor) -> Tensor = pullback(at: atTensor) { x in + foo(tensor: x, shape: Tensor([1, 3])) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([[1, 2, 3]]) - expected = atTensor - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([2]) - expected = Tensor([ - [2, 2, 2], - [2, 2, 2], - [2, 2, 2]] - ) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([ - [8, 1, 3], - [8, 1, 3], - [8, 1, 3]] - ) - expected = inputTensor + let expected = atTensor + let inputTensor: Tensor = Tensor([[1, 2, 3]]) expectEqual(expected, pb(inputTensor)) } -TensorADTests.testAllBackends("broadcast(to:)") { +TensorADTests.testAllBackends("unbroadcast(to:") { func foo(tensor: Tensor, shape: TensorShape) -> Tensor { - tensor.broadcast(to: shape) - } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,] -> [3,3] - pb = pullback(at: Tensor([99, 33, 55])) { x in - foo(tensor: x, shape: TensorShape([3, 3])) + tensor.unbroadcast(to: shape) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([3, 6, 9]) - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([ + let atTensor: Tensor = Tensor([ [1, 2, 3], [1, 2, 3], - [1, 2, 3], [1, 2, 3]] ) - expected = Tensor([4, 8, 12]) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([1, 2, 3]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // Test 4: extremely padded shape as tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([1, 2, 3]) - expectEqual(expected, pb(inputTensor)) - - // [3,1] -> [3x3] - pb = pullback(at: Tensor([[99, 33, 55]])) { x in - foo(tensor: x, shape: TensorShape([3, 3])) + let pb: (Tensor) -> Tensor = pullback(at: atTensor) { x in + foo(tensor: x, shape: TensorShape([1, 3])) } - - // Test 5: same shape as parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] - ) - expected = Tensor([[3, 6, 9]]) - expectEqual(expected, pb(inputTensor)) - - // Test 6: different shape than parameter of pullback - inputTensor = Tensor([ - [1, 2, 3], - [1, 2, 3], - [1, 2, 3], - [1, 2, 3]] + let inputTensor: Tensor = Tensor([2]) + let expected: Tensor = Tensor([ + [2, 2, 2], + [2, 2, 2], + [2, 2, 2]] ) - expected = Tensor([[4, 8, 12]]) - expectEqual(expected, pb(inputTensor)) - - // Test 7: same shape as tensor we are differentiating at - inputTensor = Tensor([[1, 2, 3]]) - expected = Tensor([[1, 2, 3]]) - expectEqual(expected, pb(inputTensor)) - - // Test 8: extremely padded shape of tensor we are differentiating at - inputTensor = Tensor([[[[[[1, 2, 3]]]]]]) - expected = Tensor([[1, 2, 3]]) expectEqual(expected, pb(inputTensor)) } -TensorADTests.testAllBackends("unbroadcast(to:") { - func foo(tensor: Tensor, shape: TensorShape) -> Tensor { - tensor.unbroadcast(to: shape) +TensorADTests.testAllBackends("unbroadcast(like:") { + func foo(tensor: Tensor, other: Tensor) -> Tensor { + tensor.unbroadcast(like: other) } - - var inputTensor: Tensor - var expected: Tensor - var pb: (Tensor) -> Tensor - - // [3,3] -> [1,3] let atTensor: Tensor = Tensor([ [1, 2, 3], [1, 2, 3], [1, 2, 3]] ) - pb = pullback(at: atTensor) { x in - foo(tensor: x, shape: TensorShape([1, 3])) + let pb: (Tensor) -> Tensor = pullback(at: atTensor) { x in + foo(tensor: x, other: Tensor([[1, 2, 3]])) } - - // Test 1: same shape as parameter of pullback - inputTensor = Tensor([[1, 2, 3]]) - expected = atTensor - expectEqual(expected, pb(inputTensor)) - - // Test 2: different shape than parameter of pullback - inputTensor = Tensor([2]) - expected = Tensor([ - [2, 2, 2], - [2, 2, 2], - [2, 2, 2]] - ) - expectEqual(expected, pb(inputTensor)) - - // Test 3: same shape as tensor we are differentiating at - inputTensor = Tensor([ + let inputTensor: Tensor = Tensor([ [8, 1, 3], [8, 1, 3], [8, 1, 3]] ) - expected = inputTensor + let expected: Tensor = inputTensor expectEqual(expected, pb(inputTensor)) }