Initial training and inference benchmarks

.gitignore

@@ -9,3 +9,7 @@
 cifar-10-batches-py/
 cifar-10-batches-bin/
 output/
+t10k-labels-idx1-ubyte
+t10k-images-idx3-ubyte
+train-labels-idx1-ubyte
+train-images-idx3-ubyte

Benchmarks/Benchmark.swift

@@ -0,0 +1,114 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+protocol BenchmarkResults: CustomStringConvertible {
+    var description: String { get }
+}
+
+struct TrainingBenchmarkResults: BenchmarkResults {
+    let iterations: Int
+    let averageTime: Double
+    let standardDeviation: Double
+
+    var description: String {
+        get {
+            return """
+                  \tAfter \(iterations) iterations:
+                  \tAverage: \(averageTime) ms, standard deviation: \(standardDeviation) ms
+                """
+        }
+    }
+}
+
+struct InferenceBenchmarkResults: BenchmarkResults {
+    let iterations: Int
+    let samplesPerSecond: Double
+    let standardDeviation: Double
+
+    var description: String {
+        get {
+            return """
+                  \tAfter \(iterations) iterations:
+                  \tSamples per second: \(samplesPerSecond), standard deviation: \(standardDeviation)
+                """
+        }
+    }
+}
+
+func timeExecution(_ operation: () -> Void) -> Double {
+    var startTime = timeval()
+    gettimeofday(&startTime, nil)
+
+    operation()
+
+    var endTime = timeval()
+    gettimeofday(&endTime, nil)
+    let secondsPortion = (endTime.tv_sec - startTime.tv_sec) * 1000
+    let microsecondsPortion = Int((endTime.tv_usec - startTime.tv_usec) / 1000)
+    return Double(secondsPortion + microsecondsPortion)
+}
+
+func statistics(for values: [Double]) -> (average: Double, standardDeviation: Double) {
+    guard values.count > 0 else { return (average: 0.0, standardDeviation: 0.0) }
+    guard values.count > 1 else { return (average: values.first!, standardDeviation: 0.0) }
+
+    let average = (values.reduce(0.0) { $0 + $1 }) / Double(values.count)
+
+    let standardDeviation = sqrt(
+        values.reduce(0.0) { $0 + ($1 - average) * ($1 - average) }
+            / Double(values.count - 1))
+
+    return (average: average, standardDeviation: standardDeviation)
+}
+
+func benchmarkTraining(iterations: Int, operation: () -> Void, callback: (BenchmarkResults) -> Void)
+{
+    var timings: [Double] = []
+    for _ in 0..<iterations {
+        timings.append(timeExecution(operation))
+    }
+
+    let (averageTime, standardDeviation) = statistics(for: timings)
+
+    let results = TrainingBenchmarkResults(
+        iterations: iterations, averageTime: averageTime, standardDeviation: standardDeviation)
+    callback(results)
+}
+
+func benchmarkInference(
+    iterations: Int, batches: Int, batchSize: Int, setup: (Int, Int) -> Void, operation: () -> Void,
+    callback: (BenchmarkResults) -> Void
+) {
+    setup(batches, batchSize)
+
+    var timings: [Double] = []
+    for _ in 0..<iterations {
+        timings.append(Double(batches * batchSize) / (timeExecution(operation) / 1000.0))
+    }
+
+    let (samplesPerSecond, standardDeviation) = statistics(for: timings)
+    let results = InferenceBenchmarkResults(
+        iterations: iterations, samplesPerSecond: samplesPerSecond,
+        standardDeviation: standardDeviation)
+    callback(results)
+}
+
+func logResults(name: String) -> (BenchmarkResults) -> Void {
+    return { results in
+        print("Benchmark: \(name):")
+        print("\(results)")
+    }
+}

Benchmarks/Models/Benchmark-LeNet.swift

@@ -0,0 +1,65 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import TensorFlow
+import Datasets
+import ImageClassificationModels
+
+final class LeNetBenchmark {
+    let epochs: Int
+    let batchSize: Int
+    let dataset: MNIST
+    var inferenceModel: LeNet!
+    var inferenceImages: Tensor<Float>!
+    var batches: Int!
+    var inferenceBatches: Int!
+
+    init(epochs: Int, batchSize: Int) {
+        self.epochs = epochs
+        self.batchSize = batchSize
+        self.dataset = MNIST(batchSize: batchSize)
+    }
+
+    func performTraining() {
+        var model = LeNet()
+        let optimizer = SGD(for: model, learningRate: 0.1)
+
+        Context.local.learningPhase = .training
+        for _ in 1...epochs {
+            for i in 0..<dataset.trainingSize / batchSize {
+                let x = dataset.trainingImages.minibatch(at: i, batchSize: batchSize)
+                let y = dataset.trainingLabels.minibatch(at: i, batchSize: batchSize)
+                let 𝛁model = model.gradient { model -> Tensor<Float> in
+                    let ŷ = model(x)
+                    return softmaxCrossEntropy(logits: ŷ, labels: y)
+                }
+                optimizer.update(&model, along: 𝛁model)
+            }
+        }
+    }
+
+    func setupInference(batches: Int, batchSize: Int) {
+        inferenceBatches = batches
+        inferenceModel = LeNet()
+        inferenceImages = Tensor<Float>(
+            randomNormal: [batchSize, 28, 28, 1], mean: Tensor<Float>(0.5),
+            standardDeviation: Tensor<Float>(0.1), seed: (0xffeffe, 0xfffe))
+    }
+
+    func performInference() {
+        for _ in 0..<inferenceBatches {
+            let _ = inferenceModel(inferenceImages)
+        }
+    }
+}

Benchmarks/README.md

@@ -0,0 +1,22 @@
+# Model benchmarks
+
+Eventually, these will contain a series of benchmarks against a variety of models in the 
+swift-models repository. The following benchmarks have been implemented:
+
+- Training LeNet against the MNIST dataset
+- Performing inference with LeNet using MNIST-sized random images
+
+These benchmarks should provide a baseline to judge performance improvements and regressions in 
+Swift for TensorFlow.
+
+## Running benchmarks
+
+To begin, you'll need the [latest version of Swift for
+TensorFlow](https://github.com/tensorflow/swift/blob/master/Installation.md)
+installed. Make sure you've added the correct version of `swift` to your path.
+
+To run all benchmarks, type the following while in the swift-models directory:
+
+```sh
+swift run -c release Benchmarks
+```

Benchmarks/main.swift

@@ -0,0 +1,23 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// LeNet-MNIST
+let leNetBenchmark = LeNetBenchmark(epochs: 1, batchSize: 128)
+benchmarkTraining(
+    iterations: 10, operation: leNetBenchmark.performTraining,
+    callback: logResults(name: "LeNet-MNIST (training)"))
+benchmarkInference(
+    iterations: 10, batches: 1000, batchSize: 1, setup: leNetBenchmark.setupInference,
+    operation: leNetBenchmark.performInference,
+    callback: logResults(name: "LeNet-MNIST (inference)"))

Package.swift

@@ -18,12 +18,14 @@ let package = Package(
         .executable(name: "MiniGoDemo", targets: ["MiniGoDemo"]),
         .library(name: "MiniGo", targets: ["MiniGo"]),
         .executable(name: "GAN", targets: ["GAN"]),
+        .executable(name: "Benchmarks", targets: ["Benchmarks"]),
     ],
     targets: [
         .target(name: "ImageClassificationModels", path: "Models/ImageClassification"),
         .target(name: "Datasets", path: "Datasets"),
         .target(name: "ModelSupport", path: "Support"),
-        .target(name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"),
+        .target(
+            name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"),
         .target(name: "Catch", path: "Catch"),
         .target(name: "Gym-FrozenLake", path: "Gym/FrozenLake"),
         .target(name: "Gym-CartPole", path: "Gym/CartPole"),
@@ -45,5 +47,9 @@ let package = Package(
         .testTarget(name: "ImageClassificationTests", dependencies: ["ImageClassificationModels"]),
         .target(name: "Transformer", path: "Transformer"),
         .target(name: "GAN", dependencies: ["Datasets", "ModelSupport"], path: "GAN"),
+        .target(
+            name: "Benchmarks",
+            dependencies: ["Datasets", "ModelSupport", "ImageClassificationModels"],
+            path: "Benchmarks"),
     ]
 )