diff --git a/.gitignore b/.gitignore index 4fea2e7e10e..87b5aaff3b9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ cifar-10-batches-py/ cifar-10-batches-bin/ output/ +t10k-labels-idx1-ubyte +t10k-images-idx3-ubyte +train-labels-idx1-ubyte +train-images-idx3-ubyte diff --git a/Benchmarks/Benchmark.swift b/Benchmarks/Benchmark.swift new file mode 100644 index 00000000000..d50f9babd3a --- /dev/null +++ b/Benchmarks/Benchmark.swift @@ -0,0 +1,104 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Foundation + +enum BenchmarkVariety { + case inferenceThroughput(batches: Int, batchSize: Int) + case trainingTime +} + +struct BenchmarkResults { + let name: String + let iterations: Int + let timings: [Double] + let variety: BenchmarkVariety +} + +extension BenchmarkResults { + var interpretedTimings: [Double] { + switch self.variety { + case let .inferenceThroughput(batches, batchSize): + return timings.map { Double(batches * batchSize) / ($0 / 1000.0) } + case .trainingTime: + return timings + } + } +} + +/// Performs the specified benchmark over a certain number of iterations and provides the result to a callback function. +func benchmark( + name: String, + iterations: Int, variety: BenchmarkVariety, + operation: () -> Void, + callback: (BenchmarkResults) -> Void +) { + var timings: [Double] = [] + for _ in 0.. Void) -> Double { + let divisor: Double = 1_000_000 + let start = Double(DispatchTime.now().uptimeNanoseconds) / divisor + body() + let end = Double(DispatchTime.now().uptimeNanoseconds) / divisor + let elapsed = end - start + return elapsed +} + +/// Provides the average and standard deviation of an array of values. +func statistics(for values: [Double]) -> (average: Double, standardDeviation: Double) { + guard values.count > 0 else { return (average: 0.0, standardDeviation: 0.0) } + guard values.count > 1 else { return (average: values.first!, standardDeviation: 0.0) } + + let average = (values.reduce(0.0) { $0 + $1 }) / Double(values.count) + + let standardDeviation = sqrt( + values.reduce(0.0) { $0 + ($1 - average) * ($1 - average) } + / Double(values.count - 1)) + + return (average: average, standardDeviation: standardDeviation) +} + +// This is a simple callback function example that only logs the result to the console. +func logResults(_ result: BenchmarkResults) { + let (average, standardDeviation) = statistics(for: result.interpretedTimings) + + switch result.variety { + case .inferenceThroughput: + print( + """ + Benchmark: \(result.name): + \tAfter \(result.iterations) iterations: + \tSamples per second: \(String(format: "%.2f", average)), standard deviation: \(String(format: "%.2f", standardDeviation)) + """) + case .trainingTime: + print( + """ + Benchmark: \(result.name): + \tAfter \(result.iterations) iterations: + \tAverage: \(String(format: "%.2f", average)) ms, standard deviation: \(String(format: "%.2f", standardDeviation)) ms + """) + } +} diff --git a/Benchmarks/Models/ImageClassificationInference.swift b/Benchmarks/Models/ImageClassificationInference.swift new file mode 100644 index 00000000000..81e787cb114 --- /dev/null +++ b/Benchmarks/Models/ImageClassificationInference.swift @@ -0,0 +1,53 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import TensorFlow +import Datasets +import ImageClassificationModels + +protocol ImageClassificationModel: Layer where Input == Tensor, Output == Tensor { + init() +} + +extension LeNet: ImageClassificationModel {} + +class ImageClassificationInference where Model: ImageClassificationModel { + // TODO: (https://github.com/tensorflow/swift-models/issues/206) Datasets should have a common + // interface to allow for them to be interchangeable in these benchmark cases. + let dataset: MNIST + var model: Model + let images: Tensor + let batches: Int + let batchSize: Int + + init(batches: Int, batchSize: Int, images: Tensor? = nil) { + self.batches = batches + self.batchSize = batchSize + self.dataset = MNIST(batchSize: batchSize) + self.model = Model() + if let providedImages = images { + self.images = providedImages + } else { + self.images = Tensor( + randomNormal: [batchSize, 28, 28, 1], mean: Tensor(0.5), + standardDeviation: Tensor(0.1), seed: (0xffeffe, 0xfffe)) + } + } + + func performInference() { + for _ in 0.. +where Model: ImageClassificationModel, Model.TangentVector.VectorSpaceScalar == Float { + // TODO: (https://github.com/tensorflow/swift-models/issues/206) Datasets should have a common + // interface to allow for them to be interchangeable in these benchmark cases. + let dataset: MNIST + let epochs: Int + let batchSize: Int + + init(epochs: Int, batchSize: Int) { + self.epochs = epochs + self.batchSize = batchSize + self.dataset = MNIST(batchSize: batchSize) + } + + func train() { + var model = Model() + // TODO: Split out the optimizer as a separate specification. + let optimizer = SGD(for: model, learningRate: 0.1) + + Context.local.learningPhase = .training + for _ in 1...epochs { + for i in 0.. Tensor in + let ŷ = model(x) + return softmaxCrossEntropy(logits: ŷ, labels: y) + } + optimizer.update(&model, along: 𝛁model) + } + } + } +} diff --git a/Benchmarks/README.md b/Benchmarks/README.md new file mode 100644 index 00000000000..18e6c352f5b --- /dev/null +++ b/Benchmarks/README.md @@ -0,0 +1,22 @@ +# Model benchmarks + +Eventually, these will contain a series of benchmarks against a variety of models in the +swift-models repository. The following benchmarks have been implemented: + +- Training LeNet against the MNIST dataset +- Performing inference with LeNet using MNIST-sized random images + +These benchmarks should provide a baseline to judge performance improvements and regressions in +Swift for TensorFlow. + +## Running benchmarks + +To begin, you'll need the [latest version of Swift for +TensorFlow](https://github.com/tensorflow/swift/blob/master/Installation.md) +installed. Make sure you've added the correct version of `swift` to your path. + +To run all benchmarks, type the following while in the swift-models directory: + +```sh +swift run -c release Benchmarks +``` \ No newline at end of file diff --git a/Benchmarks/main.swift b/Benchmarks/main.swift new file mode 100644 index 00000000000..1b2fb970116 --- /dev/null +++ b/Benchmarks/main.swift @@ -0,0 +1,29 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ImageClassificationModels + +// LeNet-MNIST +let leNetTrainingBenchmark = ImageClassificationTraining(epochs: 1, batchSize: 128) +benchmark( + name: "LeNet-MNIST (training)", + iterations: 10, variety: .trainingTime, operation: leNetTrainingBenchmark.train, + callback: logResults) + +let leNetInferenceBenchmark = ImageClassificationInference(batches: 1000, batchSize: 1) +benchmark( + name: "LeNet-MNIST (inference)", + iterations: 10, variety: .inferenceThroughput(batches: 1000, batchSize: 1), + operation: leNetInferenceBenchmark.performInference, + callback: logResults) diff --git a/Package.swift b/Package.swift index 20e140f6aad..302355d0218 100644 --- a/Package.swift +++ b/Package.swift @@ -18,12 +18,14 @@ let package = Package( .executable(name: "MiniGoDemo", targets: ["MiniGoDemo"]), .library(name: "MiniGo", targets: ["MiniGo"]), .executable(name: "GAN", targets: ["GAN"]), + .executable(name: "Benchmarks", targets: ["Benchmarks"]), ], targets: [ .target(name: "ImageClassificationModels", path: "Models/ImageClassification"), .target(name: "Datasets", path: "Datasets"), .target(name: "ModelSupport", path: "Support"), - .target(name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"), + .target( + name: "Autoencoder", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder"), .target(name: "Catch", path: "Catch"), .target(name: "Gym-FrozenLake", path: "Gym/FrozenLake"), .target(name: "Gym-CartPole", path: "Gym/CartPole"), @@ -45,5 +47,9 @@ let package = Package( .testTarget(name: "ImageClassificationTests", dependencies: ["ImageClassificationModels"]), .target(name: "Transformer", path: "Transformer"), .target(name: "GAN", dependencies: ["Datasets", "ModelSupport"], path: "GAN"), + .target( + name: "Benchmarks", + dependencies: ["Datasets", "ModelSupport", "ImageClassificationModels"], + path: "Benchmarks"), ] )