Initial training and inference benchmarks

.gitignore

@@ -9,3 +9,7 @@
 cifar-10-batches-py/
 cifar-10-batches-bin/
 output/
+t10k-labels-idx1-ubyte
+t10k-images-idx3-ubyte
+train-labels-idx1-ubyte
+train-images-idx3-ubyte

Benchmarks/Benchmark.swift

@@ -0,0 +1,104 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+enum BenchmarkVariety {
+    case inferenceThroughput(batches: Int, batchSize: Int)
+    case trainingTime
+}
+
+struct BenchmarkResults {
+    let name: String
+    let iterations: Int
+    let timings: [Double]
+    let variety: BenchmarkVariety
+}
+
+extension BenchmarkResults {
+    var interpretedTimings: [Double] {
+        switch self.variety {
+        case let .inferenceThroughput(batches, batchSize):
+            return timings.map { Double(batches * batchSize) / ($0 / 1000.0) }
+        case .trainingTime:
+            return timings
+        }
+    }
+}
+
+/// Performs the specified benchmark over a certain number of iterations and provides the result to a callback function.
+func benchmark(
+    name: String,
+    iterations: Int, variety: BenchmarkVariety,
+    operation: () -> Void,
+    callback: (BenchmarkResults) -> Void
+) {
+    var timings: [Double] = []
+    for _ in 0..<iterations {
+        let timing = time(operation)
+        timings.append(timing)
+    }
+
+    let results = BenchmarkResults(
+        name: name, iterations: iterations,
+        timings: timings, variety: variety
+    )
+    callback(results)
+}
+
+/// Returns the time elapsed while running `body` in milliseconds.
+func time(_ body: () -> Void) -> Double {
+    let divisor: Double = 1_000_000
+    let start = Double(DispatchTime.now().uptimeNanoseconds) / divisor
+    body()
+    let end = Double(DispatchTime.now().uptimeNanoseconds) / divisor
+    let elapsed = end - start
+    return elapsed
+}
+
+/// Provides the average and standard deviation of an array of values.
+func statistics(for values: [Double]) -> (average: Double, standardDeviation: Double) {
+    guard values.count > 0 else { return (average: 0.0, standardDeviation: 0.0) }
+    guard values.count > 1 else { return (average: values.first!, standardDeviation: 0.0) }
+
+    let average = (values.reduce(0.0) { $0 + $1 }) / Double(values.count)
+
+    let standardDeviation = sqrt(
+        values.reduce(0.0) { $0 + ($1 - average) * ($1 - average) }
+            / Double(values.count - 1))
+
+    return (average: average, standardDeviation: standardDeviation)
+}
+
+// This is a simple callback function example that only logs the result to the console.
+func logResults(_ result: BenchmarkResults) {
+    let (average, standardDeviation) = statistics(for: result.interpretedTimings)
+
+    switch result.variety {
+    case .inferenceThroughput:
+        print(
+            """
+                Benchmark: \(result.name):
+                \tAfter \(result.iterations) iterations:
+                \tSamples per second: \(String(format: "%.2f", average)), standard deviation: \(String(format: "%.2f", standardDeviation))
+            """)
+    case .trainingTime:
+        print(
+            """
+                Benchmark: \(result.name):
+                \tAfter \(result.iterations) iterations:
+                \tAverage: \(String(format: "%.2f", average)) ms, standard deviation: \(String(format: "%.2f", standardDeviation)) ms
+            """)
+    }
+}

Benchmarks/Models/ImageClassificationInference.swift

@@ -0,0 +1,53 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import TensorFlow
+import Datasets
+import ImageClassificationModels
+
+protocol ImageClassificationModel: Layer where Input == Tensor<Float>, Output == Tensor<Float> {
+    init()
+}
+
+extension LeNet: ImageClassificationModel {}
+
+class ImageClassificationInference<Model> where Model: ImageClassificationModel {
+    // TODO: (https://github.com/tensorflow/swift-models/issues/206) Datasets should have a common
+    // interface to allow for them to be interchangeable in these benchmark cases.
+    let dataset: MNIST
+    var model: Model
+    let images: Tensor<Float>
+    let batches: Int
+    let batchSize: Int
+
+    init(batches: Int, batchSize: Int, images: Tensor<Float>? = nil) {
+        self.batches = batches
+        self.batchSize = batchSize
+        self.dataset = MNIST(batchSize: batchSize)
+        self.model = Model()
+        if let providedImages = images {
+            self.images = providedImages
+        } else {
+            self.images = Tensor<Float>(
+                randomNormal: [batchSize, 28, 28, 1], mean: Tensor<Float>(0.5),
+                standardDeviation: Tensor<Float>(0.1), seed: (0xffeffe, 0xfffe))
+        }
+    }
+
+    func performInference() {
+        for _ in 0..<batches {
+            let _ = model(images)
+        }
+    }
+}

Benchmarks/Models/ImageClassificationTraining.swift

@@ -0,0 +1,50 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import TensorFlow
+import Datasets
+
+struct ImageClassificationTraining<Model>
+where Model: ImageClassificationModel, Model.TangentVector.VectorSpaceScalar == Float {
+    // TODO: (https://github.com/tensorflow/swift-models/issues/206) Datasets should have a common
+    // interface to allow for them to be interchangeable in these benchmark cases.
+    let dataset: MNIST
+    let epochs: Int
+    let batchSize: Int
+
+    init(epochs: Int, batchSize: Int) {
+        self.epochs = epochs
+        self.batchSize = batchSize
+        self.dataset = MNIST(batchSize: batchSize)
+    }
+
+    func train() {
+        var model = Model()
+        // TODO: Split out the optimizer as a separate specification.
+        let optimizer = SGD(for: model, learningRate: 0.1)
+
+        Context.local.learningPhase = .training
+        for _ in 1...epochs {
+            for i in 0..<dataset.trainingSize / batchSize {
+                let x = dataset.trainingImages.minibatch(at: i, batchSize: batchSize)
+                let y = dataset.trainingLabels.minibatch(at: i, batchSize: batchSize)
+                let 𝛁model = model.gradient { model -> Tensor<Float> in
+                    let ŷ = model(x)
+                    return softmaxCrossEntropy(logits: ŷ, labels: y)
+                }
+                optimizer.update(&model, along: 𝛁model)
+            }
+        }
+    }
+}

Benchmarks/README.md

@@ -0,0 +1,22 @@
+# Model benchmarks
+
+Eventually, these will contain a series of benchmarks against a variety of models in the 
+swift-models repository. The following benchmarks have been implemented:
+
+- Training LeNet against the MNIST dataset
+- Performing inference with LeNet using MNIST-sized random images
+
+These benchmarks should provide a baseline to judge performance improvements and regressions in 
+Swift for TensorFlow.
+
+## Running benchmarks
+
+To begin, you'll need the [latest version of Swift for
+TensorFlow](https://github.com/tensorflow/swift/blob/master/Installation.md)
+installed. Make sure you've added the correct version of `swift` to your path.
+
+To run all benchmarks, type the following while in the swift-models directory:
+
+```sh
+swift run -c release Benchmarks
+```

Benchmarks/main.swift

@@ -0,0 +1,29 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import ImageClassificationModels
+
+// LeNet-MNIST
+let leNetTrainingBenchmark = ImageClassificationTraining<LeNet>(epochs: 1, batchSize: 128)
+benchmark(
+    name: "LeNet-MNIST (training)",
+    iterations: 10, variety: .trainingTime, operation: leNetTrainingBenchmark.train,
+    callback: logResults)
+
+let leNetInferenceBenchmark = ImageClassificationInference<LeNet>(batches: 1000, batchSize: 1)
+benchmark(
+    name: "LeNet-MNIST (inference)",
+    iterations: 10, variety: .inferenceThroughput(batches: 1000, batchSize: 1),
+    operation: leNetInferenceBenchmark.performInference,
+    callback: logResults)

Package.swift

@@ -18,12 +18,14 @@ let package = Package(
         .executable(name: "MiniGoDemo", targets: ["MiniGoDemo"]),
         .library(name: "MiniGo", targets: ["MiniGo"]),
         .executable(name: "GAN", targets: ["GAN"]),
+        .executable(name: "Benchmarks", targets: ["Benchmarks"]),
     ],
     targets: [
         .target(name: "ImageClassificationModels", path: "Models/ImageClassification"),
         .target(name: "Datasets", path: "Datasets"),
         .target(name: "ModelSupport", path: "Support"),
-        .target(name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"),
+        .target(
+            name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"),
         .target(name: "Catch", path: "Catch"),
         .target(name: "Gym-FrozenLake", path: "Gym/FrozenLake"),
         .target(name: "Gym-CartPole", path: "Gym/CartPole"),
@@ -45,5 +47,9 @@ let package = Package(
         .testTarget(name: "ImageClassificationTests", dependencies: ["ImageClassificationModels"]),
         .target(name: "Transformer", path: "Transformer"),
         .target(name: "GAN", dependencies: ["Datasets", "ModelSupport"], path: "GAN"),
+        .target(
+            name: "Benchmarks",
+            dependencies: ["Datasets", "ModelSupport", "ImageClassificationModels"],
+            path: "Benchmarks"),
     ]
 )