From 88df455caa9e287360ed80407827f20fb71a3684 Mon Sep 17 00:00:00 2001 From: Brad Larson Date: Tue, 30 Jul 2019 15:46:30 -0500 Subject: [PATCH 1/3] Extracted CIFAR-10 dataset and ResNet models into respective modules. --- CIFAR/Helpers.swift | 51 ------ CIFAR/README.md | 18 -- CIFAR/ResNet.swift | 119 ------------ .../CIFAR10/CIFAR10.swift | 53 ++---- Datasets/CIFAR10/CIFARExample.swift | 36 ++++ Datasets/MNIST/MNIST.swift | 5 + .../Custom-CIFAR10}/Models.swift | 0 Examples/Custom-CIFAR10/README.md | 18 ++ {CIFAR => Examples/Custom-CIFAR10}/main.swift | 7 +- Examples/ResNet-CIFAR10/README.md | 18 ++ .../ResNet-CIFAR10}/main.swift | 8 +- .../DifferentiableReduce.swift | 0 .../ImageClassification}/ResNet50.swift | 171 ++++++++---------- .../ImageClassification}/ResNetV2.swift | 152 +++++++--------- .../ImageClassification}/WideResNet.swift | 57 +++--- Package.swift | 12 +- ResNet/Data.swift | 148 --------------- ResNet/README.md | 19 -- 18 files changed, 277 insertions(+), 615 deletions(-) delete mode 100644 CIFAR/Helpers.swift delete mode 100644 CIFAR/README.md delete mode 100644 CIFAR/ResNet.swift rename CIFAR/Data.swift => Datasets/CIFAR10/CIFAR10.swift (77%) create mode 100644 Datasets/CIFAR10/CIFARExample.swift rename {CIFAR => Examples/Custom-CIFAR10}/Models.swift (100%) create mode 100644 Examples/Custom-CIFAR10/README.md rename {CIFAR => Examples/Custom-CIFAR10}/main.swift (93%) create mode 100644 Examples/ResNet-CIFAR10/README.md rename {ResNet => Examples/ResNet-CIFAR10}/main.swift (92%) rename ResNet/Helpers.swift => Models/ImageClassification/DifferentiableReduce.swift (100%) rename {ResNet => Models/ImageClassification}/ResNet50.swift (69%) rename {ResNet => Models/ImageClassification}/ResNetV2.swift (57%) rename {CIFAR => Models/ImageClassification}/WideResNet.swift (83%) delete mode 100644 ResNet/Data.swift delete mode 100644 ResNet/README.md diff --git a/CIFAR/Helpers.swift b/CIFAR/Helpers.swift deleted file mode 100644 index f33301cb723..00000000000 --- a/CIFAR/Helpers.swift +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: Remove this when it's moved to the standard library. -extension Array where Element: Differentiable { - @differentiable(wrt: (self, initialResult), vjp: reduceDerivative) - func differentiableReduce( - _ initialResult: Result, - _ nextPartialResult: @differentiable (Result, Element) -> Result - ) -> Result { - return reduce(initialResult, nextPartialResult) - } - - func reduceDerivative( - _ initialResult: Result, - _ nextPartialResult: @differentiable (Result, Element) -> Result - ) -> (Result, (Result.TangentVector) -> (Array.TangentVector, Result.TangentVector)) { - var pullbacks: [(Result.TangentVector) - -> (Result.TangentVector, Element.TangentVector)] = [] - let count = self.count - pullbacks.reserveCapacity(count) - var result = initialResult - for element in self { - let (y, pb) = Swift.valueWithPullback(at: result, element, in: nextPartialResult) - result = y - pullbacks.append(pb) - } - return (value: result, pullback: { cotangent in - var resultCotangent = cotangent - var elementCotangents = TangentVector([]) - elementCotangents.base.reserveCapacity(count) - for pullback in pullbacks.reversed() { - let (newResultCotangent, elementCotangent) = pullback(resultCotangent) - resultCotangent = newResultCotangent - elementCotangents.base.append(elementCotangent) - } - return (TangentVector(elementCotangents.base.reversed()), resultCotangent) - }) - } -} diff --git a/CIFAR/README.md b/CIFAR/README.md deleted file mode 100644 index 28e853930ec..00000000000 --- a/CIFAR/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# CIFAR - -This directory contains different example convolutional networks for image -classification on the [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html) dataset. - -## Setup - -You'll need [the latest version][INSTALL] of Swift for TensorFlow -installed and added to your path. - -To train the default model, run: - -``` -cd swift-models -swift run -c release CIFAR -``` - -[INSTALL]: (https://github.com/tensorflow/swift/blob/master/Installation.md) diff --git a/CIFAR/ResNet.swift b/CIFAR/ResNet.swift deleted file mode 100644 index 9ef5100b10d..00000000000 --- a/CIFAR/ResNet.swift +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2019 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import TensorFlow - -// Original Paper: -// "Deep Residual Learning for Image Recognition" -// Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun -// https://arxiv.org/abs/1512.03385 -// using shortcut layer to connect BasicBlock layers (aka Option (B)) -// see https://github.com/akamaster/pytorch_resnet_cifar10 for explanation - -struct Conv2DBatchNorm: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var conv: Conv2D - var norm: BatchNorm - - init( - filterShape: (Int, Int, Int, Int), - strides: (Int, Int) = (1, 1) - ) { - self.conv = Conv2D(filterShape: filterShape, strides: strides, padding: .same) - self.norm = BatchNorm(featureCount: filterShape.3) - } - - @differentiable - func callAsFunction(_ input: Input) -> Output { - return input.sequenced(through: conv, norm) - } -} - -struct BasicBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var blocks: [Conv2DBatchNorm] - var shortcut: Conv2DBatchNorm - - init( - featureCounts: (Int, Int), - kernelSize: Int = 3, - strides: (Int, Int) = (2, 2), - blockCount: Int = 3 - ) { - self.blocks = [Conv2DBatchNorm( - filterShape: (kernelSize, kernelSize, featureCounts.0, featureCounts.1), - strides: strides)] - for _ in 2.. Output { - let blocksReduced = blocks.differentiableReduce(input) { last, layer in - relu(layer(last)) - } - return relu(blocksReduced + shortcut(input)) - } -} - -struct ResNet: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var inputLayer = Conv2DBatchNorm(filterShape: (3, 3, 3, 16)) - - var basicBlock1: BasicBlock - var basicBlock2: BasicBlock - var basicBlock3: BasicBlock - - init(blockCount: Int = 3) { - basicBlock1 = BasicBlock(featureCounts:(16, 16), strides: (1, 1), blockCount: blockCount) - basicBlock2 = BasicBlock(featureCounts:(16, 32), blockCount: blockCount) - basicBlock3 = BasicBlock(featureCounts:(32, 64), blockCount: blockCount) - } - - var averagePool = AvgPool2D(poolSize: (8, 8), strides: (8, 8)) - var flatten = Flatten() - var classifier = Dense(inputSize: 64, outputSize: 10, activation: softmax) - - @differentiable - func callAsFunction(_ input: Input) -> Output { - let tmp = relu(inputLayer(input)) - let convolved = tmp.sequenced(through: basicBlock1, basicBlock2, basicBlock3) - return convolved.sequenced(through: averagePool, flatten, classifier) - } -} - -extension ResNet { - enum Kind: Int { - case resNet20 = 3 - case resNet32 = 5 - case resNet44 = 7 - case resNet56 = 9 - case resNet110 = 18 - } - - init(kind: Kind) { - self.init(blockCount: kind.rawValue) - } -} diff --git a/CIFAR/Data.swift b/Datasets/CIFAR10/CIFAR10.swift similarity index 77% rename from CIFAR/Data.swift rename to Datasets/CIFAR10/CIFAR10.swift index a5f9c726ad9..ad48d1517fb 100644 --- a/CIFAR/Data.swift +++ b/Datasets/CIFAR10/CIFAR10.swift @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Original source: +// "The CIFAR-10 dataset" +// Alex Krizhevsky, Vinod Nair, and Geoffrey Hinton. +// https://www.cs.toronto.edu/~kriz/cifar.html + import Foundation import TensorFlow @@ -19,6 +24,16 @@ import TensorFlow import FoundationNetworking #endif +public struct CIFAR10 { + public let trainingDataset: Dataset + public let testDataset: Dataset + + public init() { + self.trainingDataset = Dataset(elements: loadCIFARTrainingFiles()) + self.testDataset = Dataset(elements: loadCIFARTestFile()) + } +} + func downloadCIFAR10IfNotPresent(to directory: String = ".") { let downloadPath = "\(directory)/cifar-10-batches-bin" let directoryExists = FileManager.default.fileExists(atPath: downloadPath) @@ -69,27 +84,7 @@ func downloadCIFAR10IfNotPresent(to directory: String = ".") { print("Unarchiving completed") } -struct Example: TensorGroup { - var label: Tensor - var data: Tensor - - init(label: Tensor, data: Tensor) { - self.label = label - self.data = data - } - - public init( - _handles: C - ) where C.Element: _AnyTensorHandle { - precondition(_handles.count == 2) - let labelIndex = _handles.startIndex - let dataIndex = _handles.index(labelIndex, offsetBy: 1) - label = Tensor(handle: TensorHandle(handle: _handles[labelIndex])) - data = Tensor(handle: TensorHandle(handle: _handles[dataIndex])) - } -} - -func loadCIFARFile(named name: String, in directory: String = ".") -> Example { +func loadCIFARFile(named name: String, in directory: String = ".") -> CIFARExample { downloadCIFAR10IfNotPresent(to: directory) let path = "\(directory)/cifar-10-batches-bin/\(name)" @@ -124,25 +119,17 @@ func loadCIFARFile(named name: String, in directory: String = ".") -> Example { let std = Tensor([0.229, 0.224, 0.225]) let imagesNormalized = ((imageTensor / 255.0) - mean) / std - return Example(label: Tensor(labelTensor), data: imagesNormalized) + return CIFARExample(label: Tensor(labelTensor), data: imagesNormalized) } -func loadCIFARTrainingFiles() -> Example { +func loadCIFARTrainingFiles() -> CIFARExample { let data = (1..<6).map { loadCIFARFile(named: "data_batch_\($0).bin") } - return Example( + return CIFARExample( label: Raw.concat(concatDim: Tensor(0), data.map { $0.label }), data: Raw.concat(concatDim: Tensor(0), data.map { $0.data }) ) } -func loadCIFARTestFile() -> Example { +func loadCIFARTestFile() -> CIFARExample { return loadCIFARFile(named: "test_batch.bin") } - -func loadCIFAR10() -> ( - training: Dataset, test: Dataset -) { - let trainingDataset = Dataset(elements: loadCIFARTrainingFiles()) - let testDataset = Dataset(elements: loadCIFARTestFile()) - return (training: trainingDataset, test: testDataset) -} diff --git a/Datasets/CIFAR10/CIFARExample.swift b/Datasets/CIFAR10/CIFARExample.swift new file mode 100644 index 00000000000..de9a6f05cfa --- /dev/null +++ b/Datasets/CIFAR10/CIFARExample.swift @@ -0,0 +1,36 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import TensorFlow + +public struct CIFARExample: TensorGroup { + public var label: Tensor + public var data: Tensor + + public init(label: Tensor, data: Tensor) { + self.label = label + self.data = data + } + + public init( + _handles: C + ) where C.Element: _AnyTensorHandle { + precondition(_handles.count == 2) + let labelIndex = _handles.startIndex + let dataIndex = _handles.index(labelIndex, offsetBy: 1) + label = Tensor(handle: TensorHandle(handle: _handles[labelIndex])) + data = Tensor(handle: TensorHandle(handle: _handles[dataIndex])) + } +} + diff --git a/Datasets/MNIST/MNIST.swift b/Datasets/MNIST/MNIST.swift index bc1d64c4d65..18256088a3f 100644 --- a/Datasets/MNIST/MNIST.swift +++ b/Datasets/MNIST/MNIST.swift @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Original source: +// "The MNIST database of handwritten digits" +// Yann LeCun, Corinna Cortes, and Christopher J.C. Burges +// http://yann.lecun.com/exdb/mnist/ + import Foundation import TensorFlow diff --git a/CIFAR/Models.swift b/Examples/Custom-CIFAR10/Models.swift similarity index 100% rename from CIFAR/Models.swift rename to Examples/Custom-CIFAR10/Models.swift diff --git a/Examples/Custom-CIFAR10/README.md b/Examples/Custom-CIFAR10/README.md new file mode 100644 index 00000000000..e09baafa07b --- /dev/null +++ b/Examples/Custom-CIFAR10/README.md @@ -0,0 +1,18 @@ +# CIFAR-10 with custom models + +This example demonstrates how to train the custom-defined models (based on examples from [PyTorch](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html) and [Keras](https://github.com/keras-team/keras/blob/master/examples/cifar10_cnn.py) ) against the [CIFAR-10 image classification dataset](https://www.cs.toronto.edu/~kriz/cifar.html). + +Two custom models are defined, and one is applied to an instance of the CIFAR-10 dataset. A custom training loop is defined, and the training and test losses and accuracies for each epoch are shown during training. + +## Setup + +To begin, you'll need the [latest version of Swift for +TensorFlow](https://github.com/tensorflow/swift/blob/master/Installation.md) +installed. Make sure you've added the correct version of `swift` to your path. + +To train the model, run: + +```sh +cd swift-models +swift run -c release Custom-CIFAR10 +``` diff --git a/CIFAR/main.swift b/Examples/Custom-CIFAR10/main.swift similarity index 93% rename from CIFAR/main.swift rename to Examples/Custom-CIFAR10/main.swift index f83af47e2fb..8c6f8f1ba3d 100644 --- a/CIFAR/main.swift +++ b/Examples/Custom-CIFAR10/main.swift @@ -13,11 +13,12 @@ // limitations under the License. import TensorFlow +import Datasets let batchSize = 100 -let cifarDataset = loadCIFAR10() -let testBatches = cifarDataset.test.batched(batchSize) +let dataset = CIFAR10() +let testBatches = dataset.testDataset.batched(batchSize) var model = KerasModel() let optimizer = RMSProp(for: model, learningRate: 0.0001, decay: 1e-6) @@ -28,7 +29,7 @@ Context.local.learningPhase = .training for epoch in 1...100 { var trainingLossSum: Float = 0 var trainingBatchCount = 0 - let trainingShuffled = cifarDataset.training.shuffled( + let trainingShuffled = dataset.trainingDataset.shuffled( sampleCount: 50000, randomSeed: Int64(epoch)) for batch in trainingShuffled.batched(batchSize) { let (labels, images) = (batch.label, batch.data) diff --git a/Examples/ResNet-CIFAR10/README.md b/Examples/ResNet-CIFAR10/README.md new file mode 100644 index 00000000000..de69a07d66a --- /dev/null +++ b/Examples/ResNet-CIFAR10/README.md @@ -0,0 +1,18 @@ +# ResNet-50 with CIFAR-10 + +This example demonstrates how to train the [ResNet-50 network]( https://arxiv.org/abs/1512.03385) against the [CIFAR-10 image classification dataset](https://www.cs.toronto.edu/~kriz/cifar.html). + +A modified ResNet-50 network is instantiated from the ImageClassificationModels library of standard models, and applied to an instance of the CIFAR-10 dataset. A custom training loop is defined, and the training and test losses and accuracies for each epoch are shown during training. + +## Setup + +To begin, you'll need the [latest version of Swift for +TensorFlow](https://github.com/tensorflow/swift/blob/master/Installation.md) +installed. Make sure you've added the correct version of `swift` to your path. + +To train the model, run: + +```sh +cd swift-models +swift run -c release ResNet-CIFAR10 +``` diff --git a/ResNet/main.swift b/Examples/ResNet-CIFAR10/main.swift similarity index 92% rename from ResNet/main.swift rename to Examples/ResNet-CIFAR10/main.swift index e765a6f281e..e0bd93aa22d 100644 --- a/ResNet/main.swift +++ b/Examples/ResNet-CIFAR10/main.swift @@ -13,11 +13,13 @@ // limitations under the License. import TensorFlow +import ImageClassificationModels +import Datasets let batchSize = 100 -let cifarDataset = loadCIFAR10() -let testBatches = cifarDataset.test.batched(batchSize) +let dataset = CIFAR10() +let testBatches = dataset.testDataset.batched(batchSize) // Use the network sized for CIFAR-10 var model = ResNet(inputKind: .resNet50, dataKind: .cifar) @@ -32,7 +34,7 @@ Context.local.learningPhase = .training for epoch in 1...10 { var trainingLossSum: Float = 0 var trainingBatchCount = 0 - let trainingShuffled = cifarDataset.training.shuffled( + let trainingShuffled = dataset.trainingDataset.shuffled( sampleCount: 50000, randomSeed: Int64(epoch)) for batch in trainingShuffled.batched(batchSize) { let (labels, images) = (batch.label, batch.data) diff --git a/ResNet/Helpers.swift b/Models/ImageClassification/DifferentiableReduce.swift similarity index 100% rename from ResNet/Helpers.swift rename to Models/ImageClassification/DifferentiableReduce.swift diff --git a/ResNet/ResNet50.swift b/Models/ImageClassification/ResNet50.swift similarity index 69% rename from ResNet/ResNet50.swift rename to Models/ImageClassification/ResNet50.swift index a22fd4a8099..efbb6d11181 100644 --- a/ResNet/ResNet50.swift +++ b/Models/ImageClassification/ResNet50.swift @@ -20,19 +20,16 @@ import TensorFlow // https://arxiv.org/abs/1512.03385 // using shortcut layer to connect BasicBlock layers (aka Option (B)) -enum DataKind { +public enum DataKind { case cifar case imagenet } -struct ConvBN: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct ConvBN: Layer { + public var conv: Conv2D + public var norm: BatchNorm - var conv: Conv2D - var norm: BatchNorm - - init( + public init( filterShape: (Int, Int, Int, Int), strides: (Int, Int) = (1, 1), padding: Padding = .valid @@ -42,20 +39,17 @@ struct ConvBN: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return input.sequenced(through: conv, norm) } } -struct ResidualBasicBlockShortcut: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var layer1: ConvBN - var layer2: ConvBN - var shortcut: ConvBN +public struct ResidualBasicBlockShortcut: Layer { + public var layer1: ConvBN + public var layer2: ConvBN + public var shortcut: ConvBN - init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { self.layer1 = ConvBN( filterShape: (kernelSize, kernelSize, featureCounts.0, featureCounts.1), strides: (2, 2), @@ -71,19 +65,16 @@ struct ResidualBasicBlockShortcut: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return layer2(relu(layer1(input))) + shortcut(input) } } -struct ResidualBasicBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var layer1: ConvBN - var layer2: ConvBN +public struct ResidualBasicBlock: Layer { + public var layer1: ConvBN + public var layer2: ConvBN - init( + public init( featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, strides: (Int, Int) = (1, 1) @@ -99,24 +90,21 @@ struct ResidualBasicBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return layer2(relu(layer1(input))) } } -struct ResidualBasicBlockStack: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var blocks: [ResidualBasicBlock] = [] - init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, blockCount: Int) { +public struct ResidualBasicBlockStack: Layer { + public var blocks: [ResidualBasicBlock] = [] + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, blockCount: Int) { for _ in 1.. Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let blocksReduced = blocks.differentiableReduce(input) { last, layer in layer(last) } @@ -124,16 +112,13 @@ struct ResidualBasicBlockStack: Layer { } } -struct ResidualConvBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var layer1: ConvBN - var layer2: ConvBN - var layer3: ConvBN - var shortcut: ConvBN +public struct ResidualConvBlock: Layer { + public var layer1: ConvBN + public var layer2: ConvBN + public var layer3: ConvBN + public var shortcut: ConvBN - init( + public init( featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, strides: (Int, Int) = (2, 2) @@ -152,21 +137,18 @@ struct ResidualConvBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let tmp = relu(layer2(relu(layer1(input)))) return relu(layer3(tmp) + shortcut(input)) } } -struct ResidualIdentityBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct ResidualIdentityBlock: Layer { + public var layer1: ConvBN + public var layer2: ConvBN + public var layer3: ConvBN - var layer1: ConvBN - var layer2: ConvBN - var layer3: ConvBN - - init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { self.layer1 = ConvBN(filterShape: (1, 1, featureCounts.0, featureCounts.1)) self.layer2 = ConvBN( filterShape: (kernelSize, kernelSize, featureCounts.1, featureCounts.2), @@ -175,25 +157,22 @@ struct ResidualIdentityBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let tmp = relu(layer2(relu(layer1(input)))) return relu(layer3(tmp) + input) } } -struct ResidualIdentityBlockStack: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var blocks: [ResidualIdentityBlock] = [] - init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, blockCount: Int) { +public struct ResidualIdentityBlockStack: Layer { + public var blocks: [ResidualIdentityBlock] = [] + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, blockCount: Int) { for _ in 1.. Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let blocksReduced = blocks.differentiableReduce(input) { last, layer in layer(last) } @@ -201,30 +180,27 @@ struct ResidualIdentityBlockStack: Layer { } } -struct ResNetBasic: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct ResNetBasic: Layer { + public var l1: ConvBN + public var maxPool: MaxPool2D - var l1: ConvBN - var maxPool: MaxPool2D + public var l2a = ResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + public var l2b: ResidualBasicBlockStack - var l2a = ResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - var l2b: ResidualBasicBlockStack + public var l3a = ResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) + public var l3b: ResidualBasicBlockStack - var l3a = ResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) - var l3b: ResidualBasicBlockStack + public var l4a = ResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) + public var l4b: ResidualBasicBlockStack - var l4a = ResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) - var l4b: ResidualBasicBlockStack + public var l5a = ResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) + public var l5b: ResidualBasicBlockStack - var l5a = ResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) - var l5b: ResidualBasicBlockStack + public var avgPool: AvgPool2D + public var flatten = Flatten() + public var classifier: Dense - var avgPool: AvgPool2D - var flatten = Flatten() - var classifier: Dense - - init(dataKind: DataKind, layerBlockCounts: (Int, Int, Int, Int)) { + public init(dataKind: DataKind, layerBlockCounts: (Int, Int, Int, Int)) { switch dataKind { case .imagenet: l1 = ConvBN(filterShape: (7, 7, 3, 64), strides: (2, 2), padding: .same) @@ -249,7 +225,7 @@ struct ResNetBasic: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let inputLayer = maxPool(relu(l1(input))) let level2 = inputLayer.sequenced(through: l2a, l2b) let level3 = level2.sequenced(through: l3a, l3b) @@ -259,7 +235,7 @@ struct ResNetBasic: Layer { } } -extension ResNetBasic { +public extension ResNetBasic { enum Kind { case resNet18 case resNet34 @@ -275,30 +251,27 @@ extension ResNetBasic { } } -struct ResNet: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var l1: ConvBN - var maxPool: MaxPool2D +public struct ResNet: Layer { + public var l1: ConvBN + public var maxPool: MaxPool2D - var l2a = ResidualConvBlock(featureCounts: (64, 64, 64, 256), strides: (1, 1)) - var l2b: ResidualIdentityBlockStack + public var l2a = ResidualConvBlock(featureCounts: (64, 64, 64, 256), strides: (1, 1)) + public var l2b: ResidualIdentityBlockStack - var l3a = ResidualConvBlock(featureCounts: (256, 128, 128, 512)) - var l3b: ResidualIdentityBlockStack + public var l3a = ResidualConvBlock(featureCounts: (256, 128, 128, 512)) + public var l3b: ResidualIdentityBlockStack - var l4a = ResidualConvBlock(featureCounts: (512, 256, 256, 1024)) - var l4b: ResidualIdentityBlockStack + public var l4a = ResidualConvBlock(featureCounts: (512, 256, 256, 1024)) + public var l4b: ResidualIdentityBlockStack - var l5a = ResidualConvBlock(featureCounts: (1024, 512, 512, 2048)) - var l5b: ResidualIdentityBlockStack + public var l5a = ResidualConvBlock(featureCounts: (1024, 512, 512, 2048)) + public var l5b: ResidualIdentityBlockStack - var avgPool: AvgPool2D - var flatten = Flatten() - var classifier: Dense + public var avgPool: AvgPool2D + public var flatten = Flatten() + public var classifier: Dense - init(dataKind: DataKind, layerBlockCounts: (Int, Int, Int, Int)) { + public init(dataKind: DataKind, layerBlockCounts: (Int, Int, Int, Int)) { switch dataKind { case .imagenet: l1 = ConvBN(filterShape: (7, 7, 3, 64), strides: (2, 2), padding: .same) @@ -323,7 +296,7 @@ struct ResNet: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let inputLayer = maxPool(relu(l1(input))) let level2 = inputLayer.sequenced(through: l2a, l2b) let level3 = level2.sequenced(through: l3a, l3b) @@ -333,7 +306,7 @@ struct ResNet: Layer { } } -extension ResNet { +public extension ResNet { enum Kind { case resNet50 case resNet101 diff --git a/ResNet/ResNetV2.swift b/Models/ImageClassification/ResNetV2.swift similarity index 57% rename from ResNet/ResNetV2.swift rename to Models/ImageClassification/ResNetV2.swift index cb3dbb5c90b..deb45d71130 100644 --- a/ResNet/ResNetV2.swift +++ b/Models/ImageClassification/ResNetV2.swift @@ -21,14 +21,11 @@ import TensorFlow // https://arxiv.org/abs/1603.05027 // https://github.com/KaimingHe/resnet-1k-layers/ -struct Conv2DBatchNorm: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct Conv2DBatchNorm: Layer { + public var conv: Conv2D + public var norm: BatchNorm - var conv: Conv2D - var norm: BatchNorm - - init( + public init( filterShape: (Int, Int, Int, Int), strides: (Int, Int) = (1, 1), padding: Padding = .valid @@ -38,19 +35,16 @@ struct Conv2DBatchNorm: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return input.sequenced(through: conv, norm) } } -struct BatchNormConv2D: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var norm: BatchNorm - var conv: Conv2D +public struct BatchNormConv2D: Layer { + public var norm: BatchNorm + public var conv: Conv2D - init( + public init( filterShape: (Int, Int, Int, Int), strides: (Int, Int) = (1, 1), padding: Padding = .valid @@ -60,19 +54,16 @@ struct BatchNormConv2D: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return conv(relu(norm(input))) } } -struct PreActivatedResidualBasicBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var layer1: BatchNormConv2D - var layer2: BatchNormConv2D +public struct PreActivatedResidualBasicBlock: Layer { + public var layer1: BatchNormConv2D + public var layer2: BatchNormConv2D - init( + public init( featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, strides: (Int, Int) = (1, 1) @@ -88,20 +79,17 @@ struct PreActivatedResidualBasicBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return input.sequenced(through: layer1, layer2) } } -struct PreActivatedResidualBasicBlockShortcut: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct PreActivatedResidualBasicBlockShortcut: Layer { + public var layer1: BatchNormConv2D + public var layer2: BatchNormConv2D + public var shortcut: Conv2D - var layer1: BatchNormConv2D - var layer2: BatchNormConv2D - var shortcut: Conv2D - - init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3) { self.layer1 = BatchNormConv2D( filterShape: (kernelSize, kernelSize, featureCounts.0, featureCounts.1), strides: (2, 2), @@ -117,36 +105,33 @@ struct PreActivatedResidualBasicBlockShortcut: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { return input.sequenced(through: layer1, layer2) + shortcut(input) } } -struct PreActivatedResNet18: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var l1: Conv2DBatchNorm - var maxPool: MaxPool2D +public struct PreActivatedResNet18: Layer { + public var l1: Conv2DBatchNorm + public var maxPool: MaxPool2D - var l2a = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - var l2b = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + public var l2a = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + public var l2b = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - var l3a = PreActivatedResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) - var l3b = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) + public var l3a = PreActivatedResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) + public var l3b = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) - var l4a = PreActivatedResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) - var l4b = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + public var l4a = PreActivatedResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) + public var l4b = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - var l5a = PreActivatedResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) - var l5b = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) + public var l5a = PreActivatedResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) + public var l5b = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) - var norm: BatchNorm - var avgPool: AvgPool2D - var flatten = Flatten() - var classifier: Dense + public var norm: BatchNorm + public var avgPool: AvgPool2D + public var flatten = Flatten() + public var classifier: Dense - init(imageSize: Int, classCount: Int) { + public init(imageSize: Int, classCount: Int) { // default to the ImageNet case where imageSize == 224 // Swift requires that all properties get initialized outside control flow l1 = Conv2DBatchNorm(filterShape: (7, 7, 3, 64), strides: (2, 2), padding: .same) @@ -162,7 +147,7 @@ struct PreActivatedResNet18: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let inputLayer = input.sequenced(through: l1, maxPool) let level2 = inputLayer.sequenced(through: l2a, l2b) let level3 = level2.sequenced(through: l3a, l3b) @@ -173,39 +158,36 @@ struct PreActivatedResNet18: Layer { } } -struct PreActivatedResNet34: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var l1: Conv2DBatchNorm - var maxPool: MaxPool2D - - var l2a = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - var l2b = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - var l2c = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) - - var l3a = PreActivatedResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) - var l3b = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) - var l3c = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) - var l3d = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) - - var l4a = PreActivatedResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) - var l4b = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - var l4c = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - var l4d = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - var l4e = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - var l4f = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) - - var l5a = PreActivatedResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) - var l5b = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) - var l5c = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) +public struct PreActivatedResNet34: Layer { + public var l1: Conv2DBatchNorm + public var maxPool: MaxPool2D + + public var l2a = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + public var l2b = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + public var l2c = PreActivatedResidualBasicBlock(featureCounts: (64, 64, 64, 64)) + + public var l3a = PreActivatedResidualBasicBlockShortcut(featureCounts: (64, 128, 128, 128)) + public var l3b = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) + public var l3c = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) + public var l3d = PreActivatedResidualBasicBlock(featureCounts: (128, 128, 128, 128)) + + public var l4a = PreActivatedResidualBasicBlockShortcut(featureCounts: (128, 256, 256, 256)) + public var l4b = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + public var l4c = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + public var l4d = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + public var l4e = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + public var l4f = PreActivatedResidualBasicBlock(featureCounts: (256, 256, 256, 256)) + + public var l5a = PreActivatedResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) + public var l5b = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) + public var l5c = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) - var norm: BatchNorm - var avgPool: AvgPool2D - var flatten = Flatten() - var classifier: Dense + public var norm: BatchNorm + public var avgPool: AvgPool2D + public var flatten = Flatten() + public var classifier: Dense - init(imageSize: Int, classCount: Int) { + public init(imageSize: Int, classCount: Int) { // default to the ImageNet case where imageSize == 224 // Swift requires that all properties get initialized outside control flow l1 = Conv2DBatchNorm(filterShape: (7, 7, 3, 64), strides: (2, 2), padding: .same) @@ -221,7 +203,7 @@ struct PreActivatedResNet34: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let inputLayer = input.sequenced(through: l1, maxPool) let level2 = inputLayer.sequenced(through: l2a, l2b, l2c) let level3 = level2.sequenced(through: l3a, l3b, l3c, l3d) diff --git a/CIFAR/WideResNet.swift b/Models/ImageClassification/WideResNet.swift similarity index 83% rename from CIFAR/WideResNet.swift rename to Models/ImageClassification/WideResNet.swift index 0677c0a1c82..7d518de3905 100644 --- a/CIFAR/WideResNet.swift +++ b/Models/ImageClassification/WideResNet.swift @@ -20,16 +20,13 @@ import TensorFlow // https://arxiv.org/abs/1605.07146 // https://github.com/szagoruyko/wide-residual-networks -struct BatchNormConv2DBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor +public struct BatchNormConv2DBlock: Layer { + public var norm1: BatchNorm + public var conv1: Conv2D + public var norm2: BatchNorm + public var conv2: Conv2D - var norm1: BatchNorm - var conv1: Conv2D - var norm2: BatchNorm - var conv2: Conv2D - - init( + public init( filterShape: (Int, Int, Int, Int), strides: (Int, Int) = (1, 1), padding: Padding = .same @@ -41,20 +38,17 @@ struct BatchNormConv2DBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let firstLayer = conv1(relu(norm1(input))) return conv2(relu(norm2(firstLayer))) } } -struct WideResNetBasicBlock: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var blocks: [BatchNormConv2DBlock] - var shortcut: Conv2D +public struct WideResNetBasicBlock: Layer { + public var blocks: [BatchNormConv2DBlock] + public var shortcut: Conv2D - init( + public init( featureCounts: (Int, Int), kernelSize: Int = 3, depthFactor: Int = 2, @@ -87,7 +81,7 @@ struct WideResNetBasicBlock: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let blocksReduced = blocks.differentiableReduce(input) { last, layer in relu(layer(last)) } @@ -95,22 +89,19 @@ struct WideResNetBasicBlock: Layer { } } -struct WideResNet: Layer { - typealias Input = Tensor - typealias Output = Tensor - - var l1: Conv2D +public struct WideResNet: Layer { + public var l1: Conv2D - var l2: WideResNetBasicBlock - var l3: WideResNetBasicBlock - var l4: WideResNetBasicBlock + public var l2: WideResNetBasicBlock + public var l3: WideResNetBasicBlock + public var l4: WideResNetBasicBlock - var norm: BatchNorm - var avgPool: AvgPool2D - var flatten = Flatten() - var classifier: Dense + public var norm: BatchNorm + public var avgPool: AvgPool2D + public var flatten = Flatten() + public var classifier: Dense - init(depthFactor: Int = 2, widenFactor: Int = 8) { + public init(depthFactor: Int = 2, widenFactor: Int = 8) { self.l1 = Conv2D(filterShape: (3, 3, 3, 16), strides: (1, 1), padding: .same) l2 = WideResNetBasicBlock(featureCounts: (16, 16), depthFactor: depthFactor, @@ -126,14 +117,14 @@ struct WideResNet: Layer { } @differentiable - func callAsFunction(_ input: Input) -> Output { + public func callAsFunction(_ input: Tensor) -> Tensor { let inputLayer = input.sequenced(through: l1, l2, l3, l4) let finalNorm = relu(norm(inputLayer)) return finalNorm.sequenced(through: avgPool, flatten, classifier) } } -extension WideResNet { +public extension WideResNet { enum Kind { case wideResNet16 case wideResNet16k8 diff --git a/Package.swift b/Package.swift index 094d780026d..4d8ed439bcb 100644 --- a/Package.swift +++ b/Package.swift @@ -11,9 +11,9 @@ let package = Package( products: [ .library(name: "ImageClassificationModels", targets: ["ImageClassificationModels"]), .library(name: "Datasets", targets: ["Datasets"]), + .executable(name: "Custom-CIFAR10", targets: ["Custom-CIFAR10"]), + .executable(name: "ResNet-CIFAR10", targets: ["ResNet-CIFAR10"]), .executable(name: "LeNet-MNIST", targets: ["LeNet-MNIST"]), - .executable(name: "CIFAR", targets: ["CIFAR"]), - .executable(name: "ResNet", targets: ["ResNet"]), .executable(name: "MiniGoDemo", targets: ["MiniGoDemo"]), .library(name: "MiniGo", targets: ["MiniGo"]), .executable(name: "GAN", targets: ["GAN"]), @@ -22,11 +22,16 @@ let package = Package( .target(name: "ImageClassificationModels", path: "Models/ImageClassification"), .target(name: "Datasets", path: "Datasets"), .target(name: "Autoencoder", dependencies: ["Datasets"], path: "Autoencoder"), - .target(name: "CIFAR", path: "CIFAR"), .target(name: "Catch", path: "Catch"), .target(name: "Gym-FrozenLake", path: "Gym/FrozenLake"), .target(name: "Gym-CartPole", path: "Gym/CartPole"), .target(name: "Gym-Blackjack", path: "Gym/Blackjack"), + .target( + name: "Custom-CIFAR10", dependencies: ["Datasets"], + path: "Examples/Custom-CIFAR10"), + .target( + name: "ResNet-CIFAR10", dependencies: ["ImageClassificationModels", "Datasets"], + path: "Examples/ResNet-CIFAR10"), .target( name: "LeNet-MNIST", dependencies: ["ImageClassificationModels", "Datasets"], path: "Examples/LeNet-MNIST"), @@ -35,7 +40,6 @@ let package = Package( name: "MiniGoDemo", dependencies: ["MiniGo"], path: "MiniGo", sources: ["main.swift"]), .testTarget(name: "MiniGoTests", dependencies: ["MiniGo"]), - .target(name: "ResNet", path: "ResNet"), .target(name: "Transformer", path: "Transformer"), .target(name: "GAN", dependencies: ["Datasets"], path: "GAN"), ] diff --git a/ResNet/Data.swift b/ResNet/Data.swift deleted file mode 100644 index a5f9c726ad9..00000000000 --- a/ResNet/Data.swift +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2019 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import Foundation -import TensorFlow - -#if canImport(FoundationNetworking) - import FoundationNetworking -#endif - -func downloadCIFAR10IfNotPresent(to directory: String = ".") { - let downloadPath = "\(directory)/cifar-10-batches-bin" - let directoryExists = FileManager.default.fileExists(atPath: downloadPath) - - guard !directoryExists else { return } - - print("Downloading CIFAR dataset...") - let archivePath = "\(directory)/cifar-10-binary.tar.gz" - let archiveExists = FileManager.default.fileExists(atPath: archivePath) - if !archiveExists { - print("Archive missing, downloading...") - do { - let downloadedFile = try Data( - contentsOf: URL( - string: "https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz")!) - try downloadedFile.write(to: URL(fileURLWithPath: archivePath)) - } catch { - print("Could not download CIFAR dataset, error: \(error)") - exit(-1) - } - } - - print("Archive downloaded, processing...") - - #if os(macOS) - let tarLocation = "/usr/bin/tar" - #else - let tarLocation = "/bin/tar" - #endif - - let task = Process() - task.executableURL = URL(fileURLWithPath: tarLocation) - task.arguments = ["xzf", archivePath] - do { - try task.run() - task.waitUntilExit() - } catch { - print("CIFAR extraction failed with error: \(error)") - } - - do { - try FileManager.default.removeItem(atPath: archivePath) - } catch { - print("Could not remove archive, error: \(error)") - exit(-1) - } - - print("Unarchiving completed") -} - -struct Example: TensorGroup { - var label: Tensor - var data: Tensor - - init(label: Tensor, data: Tensor) { - self.label = label - self.data = data - } - - public init( - _handles: C - ) where C.Element: _AnyTensorHandle { - precondition(_handles.count == 2) - let labelIndex = _handles.startIndex - let dataIndex = _handles.index(labelIndex, offsetBy: 1) - label = Tensor(handle: TensorHandle(handle: _handles[labelIndex])) - data = Tensor(handle: TensorHandle(handle: _handles[dataIndex])) - } -} - -func loadCIFARFile(named name: String, in directory: String = ".") -> Example { - downloadCIFAR10IfNotPresent(to: directory) - let path = "\(directory)/cifar-10-batches-bin/\(name)" - - let imageCount = 10000 - guard let fileContents = try? Data(contentsOf: URL(fileURLWithPath: path)) else { - print("Could not read dataset file: \(name)") - exit(-1) - } - guard fileContents.count == 30_730_000 else { - print( - "Dataset file \(name) should have 30730000 bytes, instead had \(fileContents.count)") - exit(-1) - } - - var bytes: [UInt8] = [] - var labels: [Int64] = [] - - let imageByteSize = 3073 - for imageIndex in 0..(shape: [imageCount], scalars: labels) - let images = Tensor(shape: [imageCount, 3, 32, 32], scalars: bytes) - - // Transpose from the CIFAR-provided N(CHW) to TF's default NHWC. - let imageTensor = Tensor(images.transposed(withPermutations: [0, 2, 3, 1])) - - let mean = Tensor([0.485, 0.456, 0.406]) - let std = Tensor([0.229, 0.224, 0.225]) - let imagesNormalized = ((imageTensor / 255.0) - mean) / std - - return Example(label: Tensor(labelTensor), data: imagesNormalized) -} - -func loadCIFARTrainingFiles() -> Example { - let data = (1..<6).map { loadCIFARFile(named: "data_batch_\($0).bin") } - return Example( - label: Raw.concat(concatDim: Tensor(0), data.map { $0.label }), - data: Raw.concat(concatDim: Tensor(0), data.map { $0.data }) - ) -} - -func loadCIFARTestFile() -> Example { - return loadCIFARFile(named: "test_batch.bin") -} - -func loadCIFAR10() -> ( - training: Dataset, test: Dataset -) { - let trainingDataset = Dataset(elements: loadCIFARTrainingFiles()) - let testDataset = Dataset(elements: loadCIFARTestFile()) - return (training: trainingDataset, test: testDataset) -} diff --git a/ResNet/README.md b/ResNet/README.md deleted file mode 100644 index 5c6b3160cb8..00000000000 --- a/ResNet/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# ResNet-50 - -This directory contains different ResNet image classification models for either ImageNet-size images -(224x224) or CIFAR-size images (32x32), and an example that trains a ResNet-50 model on the CIFAR-10 -dataset. - -## Setup - -You'll need [the latest version][INSTALL] of Swift for TensorFlow -installed and added to your path. - -To train the model on CIFAR-10, run: - -``` -cd swift-models -swift run -c release ResNet -``` - -[INSTALL]: (https://github.com/tensorflow/swift/blob/master/Installation.md) From 7a9f52af2a2f68b1e8137407c38d711861cf989d Mon Sep 17 00:00:00 2001 From: Brad Larson Date: Wed, 31 Jul 2019 15:03:19 -0500 Subject: [PATCH 2/3] Minor formatting fixes. --- Datasets/CIFAR10/CIFAR10.swift | 2 +- Datasets/CIFAR10/CIFARExample.swift | 1 - Examples/Custom-CIFAR10/main.swift | 12 ++-- Examples/ResNet-CIFAR10/main.swift | 14 +++-- .../DifferentiableReduce.swift | 30 +++++----- Models/ImageClassification/LeNet-5.swift | 2 +- Models/ImageClassification/ResNet50.swift | 42 ++++++++------ Models/ImageClassification/ResNetV2.swift | 8 +-- Models/ImageClassification/WideResNet.swift | 57 ++++++++++++------- 9 files changed, 100 insertions(+), 68 deletions(-) diff --git a/Datasets/CIFAR10/CIFAR10.swift b/Datasets/CIFAR10/CIFAR10.swift index ad48d1517fb..7b74380adbd 100644 --- a/Datasets/CIFAR10/CIFAR10.swift +++ b/Datasets/CIFAR10/CIFAR10.swift @@ -27,7 +27,7 @@ import TensorFlow public struct CIFAR10 { public let trainingDataset: Dataset public let testDataset: Dataset - + public init() { self.trainingDataset = Dataset(elements: loadCIFARTrainingFiles()) self.testDataset = Dataset(elements: loadCIFARTestFile()) diff --git a/Datasets/CIFAR10/CIFARExample.swift b/Datasets/CIFAR10/CIFARExample.swift index de9a6f05cfa..ac9bd888609 100644 --- a/Datasets/CIFAR10/CIFARExample.swift +++ b/Datasets/CIFAR10/CIFARExample.swift @@ -33,4 +33,3 @@ public struct CIFARExample: TensorGroup { data = Tensor(handle: TensorHandle(handle: _handles[dataIndex])) } } - diff --git a/Examples/Custom-CIFAR10/main.swift b/Examples/Custom-CIFAR10/main.swift index 8c6f8f1ba3d..9d01fdea794 100644 --- a/Examples/Custom-CIFAR10/main.swift +++ b/Examples/Custom-CIFAR10/main.swift @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -import TensorFlow import Datasets +import TensorFlow let batchSize = 100 @@ -53,15 +53,17 @@ for epoch in 1...100 { testBatchCount += 1 let correctPredictions = logits.argmax(squeezingAxis: 1) .== labels - correctGuessCount = correctGuessCount + - Int(Tensor(correctPredictions).sum().scalarized()) + correctGuessCount = correctGuessCount + Int( + Tensor(correctPredictions).sum().scalarized()) totalGuessCount = totalGuessCount + batchSize } let accuracy = Float(correctGuessCount) / Float(totalGuessCount) - print(""" + print( + """ [Epoch \(epoch)] \ Accuracy: \(correctGuessCount)/\(totalGuessCount) (\(accuracy)) \ Loss: \(testLossSum / Float(testBatchCount)) - """) + """ + ) } diff --git a/Examples/ResNet-CIFAR10/main.swift b/Examples/ResNet-CIFAR10/main.swift index e0bd93aa22d..be969b4d371 100644 --- a/Examples/ResNet-CIFAR10/main.swift +++ b/Examples/ResNet-CIFAR10/main.swift @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -import TensorFlow -import ImageClassificationModels import Datasets +import ImageClassificationModels +import TensorFlow let batchSize = 100 @@ -57,15 +57,17 @@ for epoch in 1...10 { testBatchCount += 1 let correctPredictions = logits.argmax(squeezingAxis: 1) .== labels - correctGuessCount = correctGuessCount + - Int(Tensor(correctPredictions).sum().scalarized()) + correctGuessCount = correctGuessCount + Int( + Tensor(correctPredictions).sum().scalarized()) totalGuessCount = totalGuessCount + batchSize } let accuracy = Float(correctGuessCount) / Float(totalGuessCount) - print(""" + print( + """ [Epoch \(epoch)] \ Accuracy: \(correctGuessCount)/\(totalGuessCount) (\(accuracy)) \ Loss: \(testLossSum / Float(testBatchCount)) - """) + """ + ) } diff --git a/Models/ImageClassification/DifferentiableReduce.swift b/Models/ImageClassification/DifferentiableReduce.swift index e82a8971477..e5021906ee0 100644 --- a/Models/ImageClassification/DifferentiableReduce.swift +++ b/Models/ImageClassification/DifferentiableReduce.swift @@ -17,16 +17,17 @@ extension Array where Element: Differentiable { @differentiable(wrt: (self, initialResult), vjp: reduceDerivative) func differentiableReduce( _ initialResult: Result, - _ nextPartialResult: @differentiable (Result, Element) -> Result + _ nextPartialResult: @differentiable(Result, Element) -> Result ) -> Result { return reduce(initialResult, nextPartialResult) } func reduceDerivative( _ initialResult: Result, - _ nextPartialResult: @differentiable (Result, Element) -> Result + _ nextPartialResult: @differentiable(Result, Element) -> Result ) -> (Result, (Result.TangentVector) -> (Array.TangentVector, Result.TangentVector)) { - var pullbacks: [(Result.TangentVector) + var pullbacks: + [(Result.TangentVector) -> (Result.TangentVector, Element.TangentVector)] = [] let count = self.count pullbacks.reserveCapacity(count) @@ -36,16 +37,19 @@ extension Array where Element: Differentiable { result = y pullbacks.append(pb) } - return (value: result, pullback: { cotangent in - var resultCotangent = cotangent - var elementCotangents = TangentVector([]) - elementCotangents.base.reserveCapacity(count) - for pullback in pullbacks.reversed() { - let (newResultCotangent, elementCotangent) = pullback(resultCotangent) - resultCotangent = newResultCotangent - elementCotangents.base.append(elementCotangent) + return ( + value: result, + pullback: { cotangent in + var resultCotangent = cotangent + var elementCotangents = TangentVector([]) + elementCotangents.base.reserveCapacity(count) + for pullback in pullbacks.reversed() { + let (newResultCotangent, elementCotangent) = pullback(resultCotangent) + resultCotangent = newResultCotangent + elementCotangents.base.append(elementCotangent) + } + return (TangentVector(elementCotangents.base.reversed()), resultCotangent) } - return (TangentVector(elementCotangents.base.reversed()), resultCotangent) - }) + ) } } diff --git a/Models/ImageClassification/LeNet-5.swift b/Models/ImageClassification/LeNet-5.swift index 6f31122c14e..fc8dba008c8 100644 --- a/Models/ImageClassification/LeNet-5.swift +++ b/Models/ImageClassification/LeNet-5.swift @@ -33,7 +33,7 @@ public struct LeNet: Layer { public var fc3 = Dense(inputSize: 84, outputSize: 10, activation: softmax) public init() {} - + @differentiable public func callAsFunction(_ input: Tensor) -> Tensor { let convolved = input.sequenced(through: conv1, pool1, conv2, pool2) diff --git a/Models/ImageClassification/ResNet50.swift b/Models/ImageClassification/ResNet50.swift index efbb6d11181..99024f2e6a0 100644 --- a/Models/ImageClassification/ResNet50.swift +++ b/Models/ImageClassification/ResNet50.swift @@ -97,6 +97,7 @@ public struct ResidualBasicBlock: Layer { public struct ResidualBasicBlockStack: Layer { public var blocks: [ResidualBasicBlock] = [] + public init(featureCounts: (Int, Int, Int, Int), kernelSize: Int = 3, blockCount: Int) { for _ in 1.. public var avgPool: AvgPool2D public var flatten = Flatten() @@ -139,7 +139,7 @@ public struct PreActivatedResNet18: Layer { avgPool = AvgPool2D(poolSize: (7, 7), strides: (7, 7)) if imageSize == 32 { l1 = Conv2DBatchNorm(filterShape: (3, 3, 3, 64), padding: .same) - maxPool = MaxPool2D(poolSize: (1, 1), strides: (1, 1)) // no-op + maxPool = MaxPool2D(poolSize: (1, 1), strides: (1, 1)) // no-op avgPool = AvgPool2D(poolSize: (4, 4), strides: (4, 4)) } norm = BatchNorm(featureCount: 512) @@ -181,7 +181,7 @@ public struct PreActivatedResNet34: Layer { public var l5a = PreActivatedResidualBasicBlockShortcut(featureCounts: (256, 512, 512, 512)) public var l5b = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) public var l5c = PreActivatedResidualBasicBlock(featureCounts: (512, 512, 512, 512)) - + public var norm: BatchNorm public var avgPool: AvgPool2D public var flatten = Flatten() @@ -195,7 +195,7 @@ public struct PreActivatedResNet34: Layer { avgPool = AvgPool2D(poolSize: (7, 7), strides: (7, 7)) if imageSize == 32 { l1 = Conv2DBatchNorm(filterShape: (3, 3, 3, 64), padding: .same) - maxPool = MaxPool2D(poolSize: (1, 1), strides: (1, 1)) // no-op + maxPool = MaxPool2D(poolSize: (1, 1), strides: (1, 1)) // no-op avgPool = AvgPool2D(poolSize: (4, 4), strides: (4, 4)) } norm = BatchNorm(featureCount: 512) diff --git a/Models/ImageClassification/WideResNet.swift b/Models/ImageClassification/WideResNet.swift index 7d518de3905..79db5cb3753 100644 --- a/Models/ImageClassification/WideResNet.swift +++ b/Models/ImageClassification/WideResNet.swift @@ -49,34 +49,46 @@ public struct WideResNetBasicBlock: Layer { public var shortcut: Conv2D public init( - featureCounts: (Int, Int), + featureCounts: (Int, Int), kernelSize: Int = 3, depthFactor: Int = 2, widenFactor: Int = 1, initialStride: (Int, Int) = (2, 2) ) { if initialStride == (1, 1) { - self.blocks = [BatchNormConv2DBlock( - filterShape: (kernelSize, kernelSize, - featureCounts.0, featureCounts.1 * widenFactor), - strides: initialStride)] + self.blocks = [ + BatchNormConv2DBlock( + filterShape: ( + kernelSize, kernelSize, + featureCounts.0, featureCounts.1 * widenFactor + ), + strides: initialStride) + ] self.shortcut = Conv2D( filterShape: (1, 1, featureCounts.0, featureCounts.1 * widenFactor), strides: initialStride) } else { - self.blocks = [BatchNormConv2DBlock( - filterShape: (kernelSize, kernelSize, - featureCounts.0 * widenFactor, featureCounts.1 * widenFactor), - strides: initialStride)] + self.blocks = [ + BatchNormConv2DBlock( + filterShape: ( + kernelSize, kernelSize, + featureCounts.0 * widenFactor, featureCounts.1 * widenFactor + ), + strides: initialStride) + ] self.shortcut = Conv2D( filterShape: (1, 1, featureCounts.0 * widenFactor, featureCounts.1 * widenFactor), strides: initialStride) } for _ in 1.. public var avgPool: AvgPool2D public var flatten = Flatten() @@ -104,13 +116,16 @@ public struct WideResNet: Layer { public init(depthFactor: Int = 2, widenFactor: Int = 8) { self.l1 = Conv2D(filterShape: (3, 3, 3, 16), strides: (1, 1), padding: .same) - l2 = WideResNetBasicBlock(featureCounts: (16, 16), depthFactor: depthFactor, + l2 = WideResNetBasicBlock( + featureCounts: (16, 16), depthFactor: depthFactor, widenFactor: widenFactor, initialStride: (1, 1)) - l3 = WideResNetBasicBlock(featureCounts: (16, 32), depthFactor: depthFactor, + l3 = WideResNetBasicBlock( + featureCounts: (16, 32), depthFactor: depthFactor, widenFactor: widenFactor) - l4 = WideResNetBasicBlock(featureCounts: (32, 64), depthFactor: depthFactor, + l4 = WideResNetBasicBlock( + featureCounts: (32, 64), depthFactor: depthFactor, widenFactor: widenFactor) - + self.norm = BatchNorm(featureCount: 64 * widenFactor) self.avgPool = AvgPool2D(poolSize: (8, 8), strides: (8, 8)) self.classifier = Dense(inputSize: 64 * widenFactor, outputSize: 10) @@ -124,8 +139,8 @@ public struct WideResNet: Layer { } } -public extension WideResNet { - enum Kind { +extension WideResNet { + public enum Kind { case wideResNet16 case wideResNet16k8 case wideResNet16k10 @@ -141,7 +156,7 @@ public extension WideResNet { case wideResNet40k8 } - init(kind: Kind) { + public init(kind: Kind) { switch kind { case .wideResNet16, .wideResNet16k8: self.init(depthFactor: 2, widenFactor: 8) From f2f22a4eca4029edfb97f190f9f932ce84f737a2 Mon Sep 17 00:00:00 2001 From: Brad Larson Date: Fri, 2 Aug 2019 07:51:07 -0500 Subject: [PATCH 3/3] Mirroring PR #187. --- .../DifferentiableReduce.swift | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 Models/ImageClassification/DifferentiableReduce.swift diff --git a/Models/ImageClassification/DifferentiableReduce.swift b/Models/ImageClassification/DifferentiableReduce.swift deleted file mode 100644 index e5021906ee0..00000000000 --- a/Models/ImageClassification/DifferentiableReduce.swift +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2019 The TensorFlow Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: Remove this when it's moved to the standard library. -extension Array where Element: Differentiable { - @differentiable(wrt: (self, initialResult), vjp: reduceDerivative) - func differentiableReduce( - _ initialResult: Result, - _ nextPartialResult: @differentiable(Result, Element) -> Result - ) -> Result { - return reduce(initialResult, nextPartialResult) - } - - func reduceDerivative( - _ initialResult: Result, - _ nextPartialResult: @differentiable(Result, Element) -> Result - ) -> (Result, (Result.TangentVector) -> (Array.TangentVector, Result.TangentVector)) { - var pullbacks: - [(Result.TangentVector) - -> (Result.TangentVector, Element.TangentVector)] = [] - let count = self.count - pullbacks.reserveCapacity(count) - var result = initialResult - for element in self { - let (y, pb) = Swift.valueWithPullback(at: result, element, in: nextPartialResult) - result = y - pullbacks.append(pb) - } - return ( - value: result, - pullback: { cotangent in - var resultCotangent = cotangent - var elementCotangents = TangentVector([]) - elementCotangents.base.reserveCapacity(count) - for pullback in pullbacks.reversed() { - let (newResultCotangent, elementCotangent) = pullback(resultCotangent) - resultCotangent = newResultCotangent - elementCotangents.base.append(elementCotangent) - } - return (TangentVector(elementCotangents.base.reversed()), resultCotangent) - } - ) - } -}