From a55402c9300f8d4bdcb6fd69baf211163f869c05 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 14:32:27 -0400 Subject: [PATCH 01/55] Minor edits. --- Sources/DeepLearning/Tensors.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Sources/DeepLearning/Tensors.swift b/Sources/DeepLearning/Tensors.swift index 1c1700649..ad6f758a6 100644 --- a/Sources/DeepLearning/Tensors.swift +++ b/Sources/DeepLearning/Tensors.swift @@ -27,18 +27,21 @@ infix operator .==: ComparisonPrecedence public extension Tensor { /// The rank of the tensor, represented as a `Tensor`. @inlinable + @_semantics("autodiff.nonvarying") var rankTensor: Tensor { return Raw.rank(self) } /// The dimensions of the tensor, represented as a `Tensor`. @inlinable + @_semantics("autodiff.nonvarying") var shapeTensor: Tensor { return Raw.shape(self) } /// The number of scalars in the tensor, represented as a `Tensor`. @inlinable + @_semantics("autodiff.nonvarying") var scalarCountTensor: Tensor { return Raw.size(self) } @@ -53,6 +56,7 @@ extension Tensor: CustomStringConvertible { /// A textual representation of the tensor. /// /// - Note: use `fullDescription` for a non-pretty-printed description showing all scalars. + @_semantics("autodiff.nonvarying") public var description: String { return array.description } @@ -69,6 +73,7 @@ public extension Tensor { /// via ellipses (`...`). /// - summarizing: If true, summarize description if element count exceeds twice /// `edgeElementCount`. + @_semantics("autodiff.nonvarying") func description( lineWidth: Int = 80, edgeElementCount: Int = 3, @@ -82,6 +87,7 @@ public extension Tensor { /// A full, non-pretty-printed textual representation of the tensor, showing /// all scalars. + @_semantics("autodiff.nonvarying") var fullDescription: String { return array.fullDescription } @@ -89,6 +95,7 @@ public extension Tensor { // Xcode Playground display conversion. extension Tensor: CustomPlaygroundDisplayConvertible { + @_semantics("autodiff.nonvarying") public var playgroundDescription: Any { return description } @@ -96,6 +103,7 @@ extension Tensor: CustomPlaygroundDisplayConvertible { // Mirror representation, used by debugger/REPL. extension Tensor: CustomReflectable { + @_semantics("autodiff.nonvarying") public var customMirror: Mirror { return Mirror(self, children: [], displayStyle: .struct) } From 874ad803edceb71122045b57a5364bcce47b09a3 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 14:35:29 -0400 Subject: [PATCH 02/55] Minor bug fix. --- Sources/DeepLearning/Operators/Math.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index 3255aea10..9a2cecebf 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -875,7 +875,7 @@ public extension Tensor where Scalar == Bool { @inlinable func all() -> Bool { let axes = Tensor(rangeFrom: 0, to: Int32(rank), stride: 1) - return _TFGetScalarOrDie(Raw.all(self, reductionIndices: axes).handle) + return Raw.all(self, reductionIndices: axes).scalarized() } /// Returns `true` if any scalars are equal to `true`. Otherwise, returns `false`. @@ -884,7 +884,7 @@ public extension Tensor where Scalar == Bool { @inlinable func any() -> Bool { let axes = Tensor(rangeFrom: 0, to: Int32(rank), stride: 1) - return _TFGetScalarOrDie(Raw.any(self, reductionIndices: axes).handle) + return Raw.any(self, reductionIndices: axes).scalarized() } /// Performs a logical AND operation along the specified axes. The reduced dimensions are From 21904cc146fbfebd0f8adbda74925d992ca2e076 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 15:43:37 -0400 Subject: [PATCH 03/55] Fixed some compiler errors. --- Sources/DeepLearning/Initializers.swift | 2 +- Sources/DeepLearning/PythonConversion.swift | 46 ++++++++++----------- Sources/DeepLearning/Tensors.swift | 42 ++++++++++++------- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/Sources/DeepLearning/Initializers.swift b/Sources/DeepLearning/Initializers.swift index 3ab3f5654..a832a86b5 100644 --- a/Sources/DeepLearning/Initializers.swift +++ b/Sources/DeepLearning/Initializers.swift @@ -205,7 +205,7 @@ internal extension Tensor where Scalar: TensorFlowFloatingPoint { alongAxis axis: Int = 0 ) -> (Tensor, (Tensor) -> Array.DifferentiableView) { let result = Tensor(concatenating: tensors, alongAxis: axis) - let posAxis = axis < 0 ? axis + tensors[0].rank: axis + let posAxis = axis < 0 ? axis + tensors[0].rank : axis let sizes = Tensor(stacking: tensors.map { $0.shapeTensor[posAxis] }) return (result, { [count = tensors.count] v in if count == 1 { return Array.DifferentiableView([v]) } diff --git a/Sources/DeepLearning/PythonConversion.swift b/Sources/DeepLearning/PythonConversion.swift index 5e52548c4..f5601ae6d 100644 --- a/Sources/DeepLearning/PythonConversion.swift +++ b/Sources/DeepLearning/PythonConversion.swift @@ -24,9 +24,9 @@ import Python // a Python import error until it is first used. private let np = Python.import("numpy") -private func debugLogNumpyError(_ message: String) { - debugLog("NumPy conversion error: " + message) -} +// private func debugLogNumpyError(_ message: String) { +// debugLog("NumPy conversion error: " + message) +// } extension ShapedArray: ConvertibleFromNumpyArray where Scalar: NumpyScalarCompatible { @@ -39,25 +39,25 @@ extension ShapedArray: ConvertibleFromNumpyArray public init?(numpy numpyArray: PythonObject) { // Check if input is a `numpy.ndarray` instance. guard Python.isinstance(numpyArray, np.ndarray) == true else { - debugLogNumpyError(""" - PythonObject input has type '\(Python.type(numpyArray))' and is not \ - an instance of 'numpy.ndarray'. - """) + // debugLogNumpyError(""" + // PythonObject input has type '\(Python.type(numpyArray))' and is not \ + // an instance of 'numpy.ndarray'. + // """) return nil } // Check if the dtype of the `ndarray` is compatible with the `Scalar` // type. guard Scalar.numpyScalarTypes.contains(numpyArray.dtype) else { - debugLogNumpyError(""" - 'numpy.ndarray' dtype '\(numpyArray.dtype)' is incompatible with \ - Swift type '\(Scalar.self)'. - """) + // debugLogNumpyError(""" + // 'numpy.ndarray' dtype '\(numpyArray.dtype)' is incompatible with \ + // Swift type '\(Scalar.self)'. + // """) return nil } let pyShape = numpyArray.__array_interface__["shape"] guard let shape = [Int](pyShape) else { - debugLogNumpyError("cannot access shape of 'numpy.ndarray' instance.") + // debugLogNumpyError("cannot access shape of 'numpy.ndarray' instance.") return nil } @@ -67,7 +67,7 @@ extension ShapedArray: ConvertibleFromNumpyArray guard let ptrVal = UInt(contiguousNumpyArray.__array_interface__["data"].tuple2.0) else { - debugLogNumpyError("cannot access data of 'numpy.ndarray' instance.") + // debugLogNumpyError("cannot access data of 'numpy.ndarray' instance.") return nil } // Note: `ptr` is not nil even if the `ndarray` is empty (i.e. has a shape @@ -103,25 +103,25 @@ extension Tensor: ConvertibleFromNumpyArray public init?(numpy numpyArray: PythonObject) { // Check if input is a `numpy.ndarray` instance. guard Python.isinstance(numpyArray, np.ndarray) == true else { - debugLogNumpyError(""" - PythonObject input has type '\(Python.type(numpyArray))' and is not \ - an instance of 'numpy.ndarray'. - """) + // debugLogNumpyError(""" + // PythonObject input has type '\(Python.type(numpyArray))' and is not \ + // an instance of 'numpy.ndarray'. + // """) return nil } // Check if the dtype of the `ndarray` is compatible with the `Scalar` // type. guard Scalar.numpyScalarTypes.contains(numpyArray.dtype) else { - debugLogNumpyError(""" - 'numpy.ndarray' dtype '\(numpyArray.dtype)' is incompatible with \ - Swift type '\(Scalar.self)'. - """) + // debugLogNumpyError(""" + // 'numpy.ndarray' dtype '\(numpyArray.dtype)' is incompatible with \ + // Swift type '\(Scalar.self)'. + // """) return nil } let pyShape = numpyArray.__array_interface__["shape"] guard let dimensions = [Int](pyShape) else { - debugLogNumpyError("cannot access shape of 'numpy.ndarray' instance.") + // debugLogNumpyError("cannot access shape of 'numpy.ndarray' instance.") return nil } let shape = TensorShape(dimensions) @@ -131,7 +131,7 @@ extension Tensor: ConvertibleFromNumpyArray let contiguousNumpyArray = np.ascontiguousarray(numpyArray) guard let ptrVal = UInt(contiguousNumpyArray.__array_interface__["data"].tuple2.0) else { - debugLogNumpyError("cannot access data of 'numpy.ndarray' instance.") + // debugLogNumpyError("cannot access data of 'numpy.ndarray' instance.") return nil } // Note: `ptr` is not nil even if the `ndarray` is empty (i.e. has a shape diff --git a/Sources/DeepLearning/Tensors.swift b/Sources/DeepLearning/Tensors.swift index ad6f758a6..7be5c5c9d 100644 --- a/Sources/DeepLearning/Tensors.swift +++ b/Sources/DeepLearning/Tensors.swift @@ -27,23 +27,29 @@ infix operator .==: ComparisonPrecedence public extension Tensor { /// The rank of the tensor, represented as a `Tensor`. @inlinable - @_semantics("autodiff.nonvarying") var rankTensor: Tensor { - return Raw.rank(self) + @_semantics("autodiff.nonvarying") + get { + return Raw.rank(self) + } } /// The dimensions of the tensor, represented as a `Tensor`. @inlinable - @_semantics("autodiff.nonvarying") var shapeTensor: Tensor { - return Raw.shape(self) + @_semantics("autodiff.nonvarying") + get { + return Raw.shape(self) + } } /// The number of scalars in the tensor, represented as a `Tensor`. @inlinable - @_semantics("autodiff.nonvarying") var scalarCountTensor: Tensor { - return Raw.size(self) + @_semantics("autodiff.nonvarying") + get { + return Raw.size(self) + } } } @@ -56,9 +62,11 @@ extension Tensor: CustomStringConvertible { /// A textual representation of the tensor. /// /// - Note: use `fullDescription` for a non-pretty-printed description showing all scalars. - @_semantics("autodiff.nonvarying") public var description: String { - return array.description + @_semantics("autodiff.nonvarying") + get { + return array.description + } } } @@ -87,25 +95,31 @@ public extension Tensor { /// A full, non-pretty-printed textual representation of the tensor, showing /// all scalars. - @_semantics("autodiff.nonvarying") var fullDescription: String { - return array.fullDescription + @_semantics("autodiff.nonvarying") + get { + return array.fullDescription + } } } // Xcode Playground display conversion. extension Tensor: CustomPlaygroundDisplayConvertible { - @_semantics("autodiff.nonvarying") public var playgroundDescription: Any { - return description + @_semantics("autodiff.nonvarying") + get { + return description + } } } // Mirror representation, used by debugger/REPL. extension Tensor: CustomReflectable { - @_semantics("autodiff.nonvarying") public var customMirror: Mirror { - return Mirror(self, children: [], displayStyle: .struct) + @_semantics("autodiff.nonvarying") + get { + return Mirror(self, children: [], displayStyle: .struct) + } } } From 8cb09bc55dab0ed3be2dd95adc5716d1be648de2 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 16:12:03 -0400 Subject: [PATCH 04/55] Added a couple of missing function overloads. --- Sources/DeepLearning/Operators/Math.swift | 42 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index 9a2cecebf..f134ddf03 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -945,13 +945,21 @@ public extension Tensor where Scalar: Numeric & Comparable { return Raw.max(self, reductionIndices: axes) } + /// Returns the maximum values along the specified axes. The reduced dimensions are removed. + /// - Parameter axes: The dimensions to reduce. + /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { + return Raw.max(self, reductionIndices: axes, keepDims: false) + } + /// Returns the maximum values along the specified axes. The reduced dimensions are removed. /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank.. Tensor { let axes = axes.map(Int32.init) - return Raw.max(self, reductionIndices: Tensor(axes), keepDims: false) + return max(squeezingAxes: Tensor(axes)) } /// Returns the maximum values along the specified axes. The reduced dimensions are removed. @@ -962,13 +970,21 @@ public extension Tensor where Scalar: Numeric & Comparable { return max(squeezingAxes: axes) } + /// Returns the minimum values along the specified axes. The reduced dimensions are removed. + /// - Parameter axes: The dimensions to reduce. + /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { + return Raw.min(self, reductionIndices: axes, keepDims: false) + } + /// Returns the minimum values along the specified axes. The reduced dimensions are removed. /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank.. Tensor { let axes = axes.map(Int32.init) - return Raw.min(self, reductionIndices: Tensor(axes), keepDims: false) + return min(squeezingAxes: Tensor(axes)) } /// Returns the minimum values along the specified axes. The reduced dimensions are removed. @@ -997,6 +1013,15 @@ public extension Tensor where Scalar: Numeric & Comparable { return Raw.argMin(self, dimension: Tensor(Int32(axis))) } + /// Returns the minimum along the specified axes. The reduced dimensions are retained with + /// value 1. + /// - Parameter axes: The dimensions to reduce. + /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { + return Raw.min(self, reductionIndices: axes, keepDims: true) + } + /// Returns the minimum along the specified axes. The reduced dimensions are retained with /// value 1. /// - Parameter axes: The dimensions to reduce. @@ -1004,7 +1029,7 @@ public extension Tensor where Scalar: Numeric & Comparable { @inlinable func min(alongAxes axes: [Int]) -> Tensor { let axes = axes.map(Int32.init) - return Raw.min(self, reductionIndices: Tensor(axes), keepDims: true) + return min(alongAxes: Tensor(axes)) } /// Returns the minimum along the specified axes. The reduced dimensions are retained with @@ -1016,6 +1041,15 @@ public extension Tensor where Scalar: Numeric & Comparable { return min(alongAxes: axes) } + /// Returns the minimum along the specified axes. The reduced dimensions are retained with + /// value 1. + /// - Parameter axes: The dimensions to reduce. + /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { + return Raw.max(self, reductionIndices: axes, keepDims: true) + } + /// Returns the minimum along the specified axes. The reduced dimensions are retained with /// value 1. /// - Parameter axes: The dimensions to reduce. @@ -1023,7 +1057,7 @@ public extension Tensor where Scalar: Numeric & Comparable { @inlinable func max(alongAxes axes: [Int]) -> Tensor { let axes = axes.map(Int32.init) - return Raw.max(self, reductionIndices: Tensor(axes), keepDims: true) + return max(alongAxes: axes) } /// Returns the minimum along the specified axes. The reduced dimensions are retained with From 84fd4419f53dd313efc9631bc105828adeb8228b Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 18:23:31 -0400 Subject: [PATCH 05/55] Bug fix. --- Sources/DeepLearning/Operators/Math.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index f134ddf03..848dc359b 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -1057,7 +1057,7 @@ public extension Tensor where Scalar: Numeric & Comparable { @inlinable func max(alongAxes axes: [Int]) -> Tensor { let axes = axes.map(Int32.init) - return max(alongAxes: axes) + return max(alongAxes: Tensor(axes)) } /// Returns the minimum along the specified axes. The reduced dimensions are retained with From c543e76a1e8f354dce44baa9ada49240d59e9ea2 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 18:57:51 -0400 Subject: [PATCH 06/55] Added VJPs for min and max. --- Sources/DeepLearning/Operators/Math.swift | 44 +++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index 848dc359b..7400ddb33 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -934,7 +934,7 @@ public extension Tensor where Scalar: Numeric & Comparable { @inlinable func min() -> Tensor { let axes = Tensor(rangeFrom: 0, to: Int32(rank), stride: 1) - return Raw.min(self, reductionIndices: axes) + return min(squeezingAxes: axes) } // NOTE: This overload is necessary, otherwise `max()` would refer to the variadic method @@ -942,13 +942,14 @@ public extension Tensor where Scalar: Numeric & Comparable { @inlinable func max() -> Tensor { let axes = Tensor(rangeFrom: 0, to: Int32(rank), stride: 1) - return Raw.max(self, reductionIndices: axes) + return max(squeezingAxes: axes) } /// Returns the maximum values along the specified axes. The reduced dimensions are removed. /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.max(self, reductionIndices: axes, keepDims: false) } @@ -974,6 +975,7 @@ public extension Tensor where Scalar: Numeric & Comparable { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.min(self, reductionIndices: axes, keepDims: false) } @@ -1018,6 +1020,7 @@ public extension Tensor where Scalar: Numeric & Comparable { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.min(self, reductionIndices: axes, keepDims: true) } @@ -1046,6 +1049,7 @@ public extension Tensor where Scalar: Numeric & Comparable { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.max(self, reductionIndices: axes, keepDims: true) } @@ -1082,6 +1086,42 @@ public extension Tensor where Scalar: Numeric & Comparable { } } +internal extension Tensor where Scalar: TensorFlowFloatingPoint { + @inlinable + func _vjpMinOrMax(squeezingAxes axes: Tensor) -> (Tensor, (Tensor) -> Tensor) { + let result = max(squeezingAxes: axes) + return (result, { v in + var y = result + var gradient = v + for i in axes.array.scalars { + y = y.expandingShape(at: Int(i)) + gradient = gradient.expandingShape(at: Int(i)) + } + + // Compute the number of selected (maximum or minimum) elements in each reduction dimension. + // If there are multiple minimum or maximum elements then the gradient will be divided between + // them. + let indicators = Tensor(y .== self) + let selectedCount = indicators.sum(alongAxes: axes) + + return gradient.broadcast(toShape: self.shapeTensor) * indicators / selectedCount + }) + } + + @inlinable + func _vjpMinOrMax(alongAxes axes: Tensor) -> (Tensor, (Tensor) -> Tensor) { + let result = max(squeezingAxes: axes) + return (result, { v in + // Compute the number of selected (maximum or minimum) elements in each reduction dimension. + // If there are multiple minimum or maximum elements then the gradient will be divided between + // them. + let indicators = Tensor(result .== self) + let selectedCount = indicators.sum(alongAxes: axes) + return v.broadcast(toShape: self.shapeTensor) * indicators / selectedCount + }) + } +} + // MARK: - Numeric Reductions public extension Tensor where Scalar: Numeric { From 262a5692f0c840d8b0ab109be3c080e92e469e92 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 18:58:38 -0400 Subject: [PATCH 07/55] Minor edits. --- Sources/DeepLearning/Operators/Math.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index 7400ddb33..ba5d664b1 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -949,7 +949,9 @@ public extension Tensor where Scalar: Numeric & Comparable { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.max(self, reductionIndices: axes, keepDims: false) } @@ -975,7 +977,9 @@ public extension Tensor where Scalar: Numeric & Comparable { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.min(self, reductionIndices: axes, keepDims: false) } From 9513dc6592f9dccdb3211966f1a993376268bdc6 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Sun, 21 Apr 2019 20:40:42 -0400 Subject: [PATCH 08/55] Bug fix. --- Sources/DeepLearning/Operators/Math.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index ba5d664b1..fa22b7190 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -1114,7 +1114,7 @@ internal extension Tensor where Scalar: TensorFlowFloatingPoint { @inlinable func _vjpMinOrMax(alongAxes axes: Tensor) -> (Tensor, (Tensor) -> Tensor) { - let result = max(squeezingAxes: axes) + let result = max(alongAxes: axes) return (result, { v in // Compute the number of selected (maximum or minimum) elements in each reduction dimension. // If there are multiple minimum or maximum elements then the gradient will be divided between From 7c8d0eabc424f825ec9b70f9f1a9c7836d3c9058 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Mon, 22 Apr 2019 00:20:27 -0400 Subject: [PATCH 09/55] Bug fix. --- Sources/DeepLearning/Operators/Math.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Operators/Math.swift b/Sources/DeepLearning/Operators/Math.swift index fa22b7190..3319b7e3d 100644 --- a/Sources/DeepLearning/Operators/Math.swift +++ b/Sources/DeepLearning/Operators/Math.swift @@ -1170,7 +1170,7 @@ public extension Tensor where Scalar: Numeric { /// - Parameter axes: The dimensions to reduce. /// - Precondition: Each value in `axes` must be in the range `-rank..) -> Tensor { return Raw.sum(self, reductionIndices: axes, keepDims: true) } From a231f6be19f677195d8231eaae0453b8dad56b0c Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Tue, 23 Apr 2019 17:01:52 -0400 Subject: [PATCH 10/55] Brought the dataset ops from the stdlib. --- Sources/DeepLearning/Operators/Dataset.swift | 224 +++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 Sources/DeepLearning/Operators/Dataset.swift diff --git a/Sources/DeepLearning/Operators/Dataset.swift b/Sources/DeepLearning/Operators/Dataset.swift new file mode 100644 index 000000000..40ca4c7cc --- /dev/null +++ b/Sources/DeepLearning/Operators/Dataset.swift @@ -0,0 +1,224 @@ +// Copyright 2018 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if !COMPILING_TENSORFLOW_MODULE +import TensorFlow +#endif + +/// The default graph seed. +/// +/// - Note: See TensorFlow's `python.framework.random_seed.DEFAULT_GRAPH_SEED`. +@usableFromInline let _defaultGraphSeed: Int64 = 87654321 + +/// Returns the local seeds an operation should use given an op-specific seed. +/// +/// Given operation-specific seed, `seed`, this helper function returns two +/// seeds derived from graph-level and op-level seeds. Many random operations +/// internally use the two seeds to allow user to change the seed globally for a +/// graph, or for only specific operations. +/// +/// - Note: See TensorFlow's `python.framework.random_seed.get_seed`. +/// +// TODO: There's no support for TF's "global seed" yet, so we always use the +// default graph seed as the first seed. Need to investigate the best way to +// model TF's "global seed". +@usableFromInline @inline(__always) +func _tensorSeeds(_ seed: Tensor) -> (Tensor, Tensor) { + return (Tensor(_defaultGraphSeed), seed) +} + +//===----------------------------------------------------------------------===// +// Single value dataset +//===----------------------------------------------------------------------===// + +/// Represents a potentially large set of elements. +/// +/// A `Dataset` can be used to represent an input pipeline as a collection of +/// element tensors. +@_fixed_layout +public struct Dataset { + public let _handle: VariantHandle + + @inlinable @inline(__always) + public init(_handle: VariantHandle) { + self._handle = _handle + } +} + +public extension Dataset { + @inlinable @inline(__always) + init(randomSeed: Int64) { + let (seed1, seed2) = _tensorSeeds(Tensor(randomSeed)) + self.init(_handle: Raw.experimentalRandomDataset( + seed: seed1, + seed2: seed2, + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } +} + +public extension Dataset { + /// Creates a dataset from a batch of elements as a tensor. + @inlinable @inline(__always) + init(elements: Element) { + // A dataset creation op only runs on TF CPU. + self.init(_handle: Raw.tensorSliceDataset( + components: elements, + outputShapes: Element._unknownShapeList)) + } +} + +extension Dataset: Sequence { + public typealias Iterator = DatasetIterator + + /// Returns an iterator over the elements of this dataset. + @inlinable @inline(__always) + public func makeIterator() -> DatasetIterator { + let resource = Raw.anonymousIterator( + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList) + Raw.makeIterator(dataset: _handle, iterator: resource) + return DatasetIterator(_handle: resource) + } +} + +public extension Dataset { + // Note that this Dataset API implementation uses an experimental tracing + // feature, which is not robust and does not have great diagnostics yet. + @inlinable @inline(__always) + func map( + _ transform: (Element) -> ResultElement + ) -> Dataset { + return Dataset(_handle: Raw.mapDataset( + inputDataset: _handle, + otherArguments: Tensor(0), + f: transform, + outputTypes: ResultElement._typeList, + outputShapes: ResultElement._unknownShapeList, + useInterOpParallelism: true, + preserveCardinality: false)) + } + + @inlinable @inline(__always) + func filter( + _ isIncluded: (Element) -> Tensor + ) -> Dataset { + return Dataset(_handle: Raw.filterDataset( + inputDataset: _handle, + otherArguments: Tensor(0), + predicate: isIncluded, + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } +} + +public extension Dataset { + @inlinable @inline(__always) + func prefetched(count: Int) -> Dataset { + return Dataset(_handle: Raw.prefetchDataset( + inputDataset: _handle, + bufferSize: Tensor(Int64(count)), + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } + + @inlinable @inline(__always) + func shuffled( + sampleCount: Int, + randomSeed: Int64, + reshuffleForEachIterator: Bool = true + ) -> Dataset { + let (seed1, seed2) = _tensorSeeds(Tensor(randomSeed)) + return Dataset(_handle: Raw.shuffleDataset( + inputDataset: _handle, + bufferSize: Tensor(Int64(sampleCount)), + seed: seed1, + seed2: seed2, + reshuffleEachIteration: reshuffleForEachIterator, + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } + + @inlinable @inline(__always) + func batched(_ batchSize: Int) -> Dataset { + return Dataset(_handle: Raw.batchDataset( + inputDataset: _handle, + batchSize: Tensor(Int64(batchSize)), + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } + + @inlinable @inline(__always) + func repeated(count: Int? = nil) -> Dataset { + return Dataset(_handle: Raw.repeatDataset( + inputDataset: _handle, + count: Tensor(Int64(count ?? -1)), + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList)) + } +} + +/// The type that allows iteration over a dataset's elements. +@_fixed_layout +public struct DatasetIterator { + @usableFromInline let _handle: ResourceHandle + + @usableFromInline @inline(__always) + internal init(_handle: ResourceHandle) { + self._handle = _handle + } +} + +extension DatasetIterator: IteratorProtocol { + /// Advances to the next element and returns it, or `nil` if no next element + /// exists. + @inlinable @inline(__always) + public mutating func next() -> Element? { + let optional = Raw.iteratorGetNextAsOptional( + iterator: _handle, + outputTypes: Element._typeList, + outputShapes: Element._unknownShapeList) + guard Raw.optionalHasValue(optional: optional).scalarized() else { + return nil + } + return Raw.optionalGetValue( + optional: optional, + outputShapes: Element._unknownShapeList) + } +} + +/// A 2-tuple-like struct that conforms to TensorGroup that represents a tuple +/// of 2 types conforming to TensorGroup. +@_fixed_layout +public struct Zip2TensorGroup: TensorGroup { + public var first: T + public var second: U + + public init(_ first: T, _ second: U) { + self.first = first + self.second = second + } +} + +// TODO(SR-9156): This does not work in graph mode. +@inlinable @inline(__always) +public func zip( + _ dataset1: Dataset, _ dataset2: Dataset +) -> Dataset> { + let handle = Raw.zipDataset( + inputDatasets: [dataset1._handle, dataset2._handle], + outputTypes: Zip2TensorGroup._typeList, + outputShapes: Zip2TensorGroup._unknownShapeList) + return Dataset(_handle: handle) +} From a7b98a8fd9bd5dbf07960e54e7c90fad633fe3b6 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Wed, 24 Apr 2019 21:30:51 -0400 Subject: [PATCH 11/55] TensorFlow/TensorFlowCore refactoring. --- Sources/DeepLearning/Context.swift | 4 +- .../DeepLearning/DifferentialOperators.swift | 4 +- Sources/DeepLearning/Helpers.swift | 4 +- Sources/DeepLearning/Initializers.swift | 43 +- Sources/DeepLearning/Layer.swift | 534 +++++++++--------- Sources/DeepLearning/Loss.swift | 4 +- Sources/DeepLearning/Operators/Basic.swift | 22 +- .../DeepLearning/Operators/Comparison.swift | 79 ++- Sources/DeepLearning/Operators/Dataset.swift | 6 +- Sources/DeepLearning/Operators/Math.swift | 125 +--- Sources/DeepLearning/Operators/NN.swift | 4 +- Sources/DeepLearning/Optimizer.swift | 4 +- Sources/DeepLearning/PythonConversion.swift | 4 +- .../{Tensors.swift => Tensor.swift} | 41 +- 14 files changed, 377 insertions(+), 501 deletions(-) rename Sources/DeepLearning/{Tensors.swift => Tensor.swift} (78%) diff --git a/Sources/DeepLearning/Context.swift b/Sources/DeepLearning/Context.swift index 266ab2902..c607e902e 100644 --- a/Sources/DeepLearning/Context.swift +++ b/Sources/DeepLearning/Context.swift @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !COMPILING_TENSORFLOW_MODULE -import TensorFlow -#endif +import TensorFlowCore #if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) import Darwin diff --git a/Sources/DeepLearning/DifferentialOperators.swift b/Sources/DeepLearning/DifferentialOperators.swift index bfb53db77..e499e737e 100644 --- a/Sources/DeepLearning/DifferentialOperators.swift +++ b/Sources/DeepLearning/DifferentialOperators.swift @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !COMPILING_TENSORFLOW_MODULE -import TensorFlow -#endif +import TensorFlowCore //===------------------------------------------------------------------------------------------===// // Method-style Differential Operators diff --git a/Sources/DeepLearning/Helpers.swift b/Sources/DeepLearning/Helpers.swift index 4d9c0217b..1d56b0f76 100644 --- a/Sources/DeepLearning/Helpers.swift +++ b/Sources/DeepLearning/Helpers.swift @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !COMPILING_TENSORFLOW_MODULE -@_exported import TensorFlow -#endif +@_exported import TensorFlowCore /// Returns a tensor with the same shape and scalars as the specified tensor. @inlinable diff --git a/Sources/DeepLearning/Initializers.swift b/Sources/DeepLearning/Initializers.swift index a832a86b5..6a35ef3f6 100644 --- a/Sources/DeepLearning/Initializers.swift +++ b/Sources/DeepLearning/Initializers.swift @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !COMPILING_TENSORFLOW_MODULE -import TensorFlow -#endif +import TensorFlowCore public extension Tensor { /// Creates a tensor with the specified shape and a single, repeated scalar @@ -23,7 +21,7 @@ public extension Tensor { /// - Parameters: /// - shape: The dimensions of the tensor. /// - repeatedValue: The scalar value to repeat. - @inlinable + @inlinable @inline(__always) @available(*, deprecated, renamed: "init(repeating:shape:)") init(shape: TensorShape, repeating repeatedValue: Scalar) { self.init(repeating: repeatedValue, shape: shape) @@ -34,7 +32,7 @@ public extension Tensor { /// - Parameters: /// - repeatedValue: The scalar value to repeat. /// - shape: The dimensions of the tensor. - @inlinable + @inlinable @inline(__always) @differentiable( vjp: _vjpInit(repeating:shape:) where Scalar: TensorFlowFloatingPoint) init(repeating repeatedValue: Scalar, shape: TensorShape) { @@ -45,7 +43,7 @@ public extension Tensor { /// Creates a tensor by broadcasting the given scalar to a given rank with /// all dimensions being 1. - @inlinable + @inlinable @inline(__always) // @differentiable(where Scalar: TensorFlowFloatingPoint) init(broadcasting scalar: Scalar, rank: Int) { self = Tensor(scalar).reshaped(to: TensorShape(repeating: 1, count: rank)) @@ -54,7 +52,7 @@ public extension Tensor { /// Creates a tensor of shape `[4]` from a 4-tuple. /// - Note: This is intended for internal use, for example, to initialize a /// tensor attribute from `convolved2D`'s `strides` argument. - @inlinable + @inlinable @inline(__always) internal init(_ scalars: (Scalar, Scalar, Scalar, Scalar)) { self.init([scalars.0, scalars.1, scalars.2, scalars.3]) } @@ -78,13 +76,13 @@ internal extension Tensor where Scalar: TensorFlowFloatingPoint { public extension Tensor where Scalar: Numeric { /// Perform an element-wise type conversion from a `Bool` tensor. - @inlinable + @inlinable @inline(__always) init(_ other: Tensor) { self = Raw.cast(other) } /// Perform an element-wise conversion from another `Tensor`. - @inlinable + @inlinable @inline(__always) @differentiable( vjp: _vjpCast where Scalar: TensorFlowFloatingPoint, OtherScalar: TensorFlowFloatingPoint) init(_ other: Tensor) { @@ -107,10 +105,10 @@ internal extension Tensor where Scalar: TensorFlowFloatingPoint { public extension Tensor { /// Creates a tensor from an array of tensors (which may themselves be scalars). - @inlinable - @differentiable(where Scalar: TensorFlowFloatingPoint) + @inlinable @inline(__always) + @differentiable(vjp: _vjpInitElements where Scalar: TensorFlowFloatingPoint) init(_ elements: [Tensor]) { - self = Tensor(stacking: elements) + self = Raw.pack(elements) } /// Stacks `tensors`, along the `axis` dimension, into a new tensor with rank one higher than @@ -141,7 +139,7 @@ public extension Tensor { /// provided tensors. /// /// - Returns: The stacked tensor. - @inlinable + @inlinable @inline(__always) @differentiable(vjp: _vjpStacking where Scalar: TensorFlowFloatingPoint) init(stacking tensors: [Tensor], alongAxis axis: Int = 0) { self = Raw.pack(tensors, axis: Int64(axis)) @@ -179,7 +177,7 @@ public extension Tensor { /// provided tensors. /// /// - Returns: The concatenated tensor. - @inlinable + @inlinable @inline(__always) @differentiable(vjp: _vjpConcatenating where Scalar: TensorFlowFloatingPoint) init(concatenating tensors: [Tensor], alongAxis axis: Int = 0) { precondition(tensors.count > 0) @@ -188,6 +186,13 @@ public extension Tensor { } internal extension Tensor where Scalar: TensorFlowFloatingPoint { + @inlinable + static func _vjpInitElements( + _ elements: [Tensor] + ) -> (Tensor, (Tensor) -> Array.DifferentiableView) { + return _vjpStacking(stacking: elements) + } + @inlinable static func _vjpStacking( stacking tensors: [Tensor], @@ -223,7 +228,7 @@ public extension Tensor where Scalar: Numeric { /// Creates a tensor with all scalars set to zero. /// /// - Parameter shape: Shape of the tensor. - @inlinable + @inlinable @inline(__always) init(zeros shape: TensorShape) { self.init(repeating: 0, shape: shape) } @@ -231,7 +236,7 @@ public extension Tensor where Scalar: Numeric { /// Creates a tensor with all scalars set to one. /// /// - Parameter shape: Shape of the tensor. - @inlinable + @inlinable @inline(__always) init(ones shape: TensorShape) { self.init(repeating: 1, shape: shape) } @@ -246,8 +251,7 @@ public extension Tensor where Scalar: Numeric { /// the resulting sequence. /// - stride: The amount to step by with each iteration. `stride` must be /// positive. - /// - @inlinable + @inlinable @inline(__always) init(rangeFrom start: Scalar, to end: Scalar, stride: Scalar) { self = Raw.range(start: Tensor(start), limit: Tensor(end), delta: Tensor(stride)) } @@ -279,8 +283,7 @@ public extension Tensor where Scalar: Numeric { /// - offValue: A scalar defining the value at a location that is not /// referred to by any index in `indices`. /// - axis: The axis to fill. The default is `-1`, a new inner-most axis. - /// - @inlinable + @inlinable @inline(__always) init( oneHotAtIndices indices: Tensor, depth: Int, diff --git a/Sources/DeepLearning/Layer.swift b/Sources/DeepLearning/Layer.swift index 774521a18..81eb59444 100644 --- a/Sources/DeepLearning/Layer.swift +++ b/Sources/DeepLearning/Layer.swift @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !COMPILING_TENSORFLOW_MODULE -import TensorFlow -#endif +import TensorFlowCore /// A neural network layer. /// @@ -1226,269 +1224,269 @@ public struct Reshape: Layer { } } -/// An input to a recurrent neural network. -public struct RNNCellInput: Differentiable { - /// The input at the current time step. - public var input: Input - /// The previous state. - public var state: State - - @differentiable - public init(input: Input, state: State) { - self.input = input - self.state = state - } -} - -/// An output to a recurrent neural network. -public struct RNNCellOutput: Differentiable { - /// The output at the current time step. - public var output: Output - /// The current state. - public var state: State - - @differentiable - public init(output: Output, state: State) { - self.output = output - self.state = state - } -} - -/// A recurrent neural network cell. -public protocol RNNCell: Layer where Input == RNNCellInput, - Output == RNNCellOutput { - /// The input at a time step. - associatedtype TimeStepInput: Differentiable - /// The output at a time step. - associatedtype TimeStepOutput: Differentiable - /// The state that may be preserved across time steps. - associatedtype State: Differentiable - /// The zero state. - var zeroState: State { get } -} - -public extension RNNCell { - /// Returns the new state obtained from applying the RNN cell to the input at the current time - /// step and the previous state. - /// - /// - Parameters: - /// - timeStepInput: The input at the current time step. - /// - previousState: The previous state of the RNN cell. - /// - Returns: The output. - @differentiable - func call(input: TimeStepInput, state: State) -> RNNCellOutput { - return self(RNNCellInput(input: input, state: state)) - } -} - -/// A Simple RNN Cell. -public struct SimpleRNNCell: RNNCell, VectorNumeric { - public var weight: Tensor - public var bias: Tensor - - @noDerivative public var stateShape: TensorShape { - return TensorShape([1, weight.shape[1]]) - } - - public var zeroState: Tensor { - return Tensor(zeros: stateShape) - } - - public typealias State = Tensor - public typealias TimeStepInput = Tensor - public typealias TimeStepOutput = State - public typealias Input = RNNCellInput - public typealias Output = RNNCellOutput - - /// Creates a `SimpleRNNCell` with the specified input size and hidden state size. - /// - /// - Parameters: - /// - inputSize: The number of features in 2-D input tensors. - /// - hiddenSize: The number of features in 2-D hidden states. - /// - seed: The random seed for initialization. The default value is random. - public init(inputSize: Int, hiddenSize: Int, - seed: (Int64, Int64) = (Int64.random(in: Int64.min..