-
Notifications
You must be signed in to change notification settings - Fork 1
/
SlimMPSCNN.swift
executable file
·492 lines (393 loc) · 22.5 KB
/
SlimMPSCNN.swift
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
//
// SlimMPSCNN.swift
// MPSCNNfeeder
//
// Created by Kazu Komoto on 11/25/16.
// Copyright © 2016 Kazu Komoto. All rights reserved.
//
//
/*
The MIT License (MIT)
Copyright (c) 2016 Kazu Komoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/*
Abstract:
This file describes slimmer routines to create some common MPSCNNFunctions,
it is useful especially to fetch network parameters from .dat files or HDF5 file.
This is based on SlimMPSCNN.swift provided by Apple.
*/
import Foundation
import MetalPerformanceShaders
import HDF5Kit
/**
This depends on MetalPerformanceShaders.framework
The SlimMPSCNNConvolution is a wrapper class around MPSCNNConvolution used to encapsulate:
- making an MPSCNNConvolutionDescriptor,
- adding network parameters (weights and bias binaries by memory mapping the binaries)
- getting our convolution layer
*/
class SlimMPSCNNConvolution: MPSCNNConvolution{
/**
A property to keep info from init time whether we will pad input image or not for use during encode call
*/
private var padding = true
/**
Initializes a fully connected kernel.
- Parameters:
- kernelWidth: Kernel Width
- kernelHeight: Kernel Height
- inputFeatureChannels: Number feature channels in input of this layer
- outputFeatureChannels: Number feature channels from output of this layer
- neuronFilter: A neuronFilter to add at the end as activation, default is nil
- device: The MTLDevice on which this SlimMPSCNNConvolution filter will be used
- kernelParamsBinaryName: name of the layer to fetch kernelParameters by adding a prefix "weights_" or "bias_"
- padding: Bool value whether to use padding or not
- strideXY: Stride of the filter
- destinationFeatureChannelOffset: FeatureChannel no. in the destination MPSImage to start writing from, helps with concat operations
- groupNum: if grouping is used, default value is 1 meaning no groups
- Returns:
A valid SlimMPSCNNConvolution object or nil, if failure.
*/
init(kernelWidth: UInt, kernelHeight: UInt, inputFeatureChannels: UInt, outputFeatureChannels: UInt, neuronFilter: MPSCNNNeuron? = nil, device: MTLDevice, kernelParamsBinaryName: String, padding willPad: Bool = true, strideXY: (UInt, UInt) = (1, 1), destinationFeatureChannelOffset: UInt = 0, groupNum: UInt = 1){
// check whether dat file exists in Bundle or Temporary directory in the client device
let fileManager = FileManager.default
var wtPath: String?
var bsPath: String?
let wtPathBundle = Bundle.main.path(forResource: "weights_" + kernelParamsBinaryName, ofType: "dat")
let bsPathBundle = Bundle.main.path(forResource: "bias_" + kernelParamsBinaryName, ofType: "dat")
// Note: Switch from Temporary directory to Library/Caches/ if needed to persist the data after closing)
let tmpPath = URL(fileURLWithPath: NSTemporaryDirectory())
let wtPathTmp = tmpPath.appendingPathComponent("weights_" + kernelParamsBinaryName).appendingPathExtension("dat")
let bsPathTmp = tmpPath.appendingPathComponent("bias_" + kernelParamsBinaryName).appendingPathExtension("dat")
// if fileManager.fileExists(atPath:wtPathBundle!) && fileManager.fileExists(atPath:bsPathBundle!) {
if (wtPathBundle != nil) && (bsPathBundle != nil) {
// Use parameters in Bundle
wtPath = wtPathBundle
bsPath = bsPathBundle
} else if fileManager.fileExists(atPath:wtPathTmp.path) && fileManager.fileExists(atPath:bsPathTmp.path) {
// Use parameters in Tmp
wtPath = wtPathTmp.path
bsPath = bsPathTmp.path
} else {
// Read parameters from HDF5 file and store to dat file in Tmp directory
// MARK: Parse HDF5 file
guard let path = Bundle.main.path(forResource: "mnist_model_weights", ofType: "h5") else {
fatalError("Failed to get a path")
}
guard let file = File.open(path, mode: .readOnly) else {
fatalError("Failed to open file at \(path)")
}
guard let layerNamesStringAttribute = file.openStringAttribute("layer_names") else {
fatalError("Failed to open attribute 'layer_names'")
}
guard let layerNames = try? layerNamesStringAttribute.read() else {
fatalError("Failed to get layer names")
}
// guard let layerNames = file.openGroup("/")?.objectNames() else {
// fatalError("Failed to open group name '/' and failed to get layer names")
// }
// count used for file name later
var countOfConvLayer = 0
var countOfFcLayer = 0
var partOfFileName = ""
for layerName in layerNames {
guard let layerGroup = file.openGroup(layerName) else {
fatalError("Failed to open group of \(layerName)")
}
if layerGroup.objectNames().count > 0 {
// only the layer that has parameters remain
guard let wtDataset = layerGroup.openFloatDataset(layerName + "_W:0") else {
fatalError("Failed to open data set of \(layerName)_W:0")
}
guard let bsDataset = layerGroup.openFloatDataset(layerName + "_b:0") else {
fatalError("Failed to open data set of \(layerName)_b:0")
}
var dimension = wtDataset.space.dims
guard var wtArray = try? wtDataset.read() else {
fatalError("Failed to read data set of \(layerName)_W:0")
}
guard var bsArray = try? bsDataset.read() else {
fatalError("Failed to read data set of \(layerName)_b:0")
}
let wtLength = wtArray.count
let bsLength = bsArray.count
if dimension.count == 4 {
// weights for convolution layer
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 2, axis2: 3, dimensionOfArray: &dimension)
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 1, axis2: 2, dimensionOfArray: &dimension)
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 0, axis2: 1, dimensionOfArray: &dimension)
countOfConvLayer += 1
partOfFileName = "conv" + String(countOfConvLayer)
} else if dimension.count == 2 {
// weights for fully connected layer
wtArray = SwapAxes.for2dFlatArray(originalArray: wtArray, axis1: 0, axis2: 1, dimensionOfArray: &dimension)
countOfFcLayer += 1
partOfFileName = "fc" + String(countOfFcLayer)
} else {
fatalError("Dataset's dimension is neither 4 (convolution layer) nor 2 (fully connected layer)")
}
let wtData = NSData(bytes: &wtArray, length: wtLength * MemoryLayout<Float>.size)
let bsData = NSData(bytes: &bsArray, length: bsLength * MemoryLayout<Float>.size)
// Save to temporary directory (Note: Switch to Library/Caches/ if you want to persist the data)
let filePath = URL(fileURLWithPath: NSTemporaryDirectory())
let wtFilePath = filePath.appendingPathComponent("weights_" + partOfFileName).appendingPathExtension("dat")
let bsFilePath = filePath.appendingPathComponent("bias_" + partOfFileName).appendingPathExtension("dat")
do {
try wtData.write(to: wtFilePath, options: .atomic)
} catch {
print(error)
}
do {
try bsData.write(to: bsFilePath, options: .atomic)
} catch {
print(error)
}
}
}
wtPath = wtPathTmp.path
bsPath = bsPathTmp.path
}
// calculate the size of weights and bias required to be memory mapped into memory
let sizeBias = outputFeatureChannels * UInt(MemoryLayout<Float>.size)
let sizeWeights = inputFeatureChannels * kernelHeight * kernelWidth * outputFeatureChannels * UInt(MemoryLayout<Float>.size)
// // get the url to this layer's weights and bias
// let wtPath = Bundle.main.path(forResource: "weights_" + kernelParamsBinaryName, ofType: "dat")
// let bsPath = Bundle.main.path(forResource: "bias_" + kernelParamsBinaryName, ofType: "dat")
// open file descriptors in read-only mode to parameter files
let fd_w = open( wtPath!, O_RDONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
let fd_b = open( bsPath!, O_RDONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
assert(fd_w != -1, "Error: failed to open output file at \""+wtPath!+"\" errno = \(errno)\n")
assert(fd_b != -1, "Error: failed to open output file at \""+bsPath!+"\" errno = \(errno)\n")
// memory map the parameters
let hdrW = mmap(nil, Int(sizeWeights), PROT_READ, MAP_FILE | MAP_SHARED, fd_w, 0)
let hdrB = mmap(nil, Int(sizeBias), PROT_READ, MAP_FILE | MAP_SHARED, fd_b, 0)
// cast Void pointers to Float
let w = UnsafePointer<Float>(hdrW!.assumingMemoryBound(to: Float.self))
let b = UnsafePointer<Float>(hdrB!.assumingMemoryBound(to: Float.self))
assert(w != UnsafePointer<Float>(bitPattern: -1), "mmap failed with errno = \(errno)")
assert(b != UnsafePointer<Float>(bitPattern: -1), "mmap failed with errno = \(errno)")
// create appropriate convolution descriptor with appropriate stride
let convDesc = MPSCNNConvolutionDescriptor(kernelWidth: Int(kernelWidth),
kernelHeight: Int(kernelHeight),
inputFeatureChannels: Int(inputFeatureChannels),
outputFeatureChannels: Int(outputFeatureChannels),
neuronFilter: neuronFilter)
convDesc.strideInPixelsX = Int(strideXY.0)
convDesc.strideInPixelsY = Int(strideXY.1)
assert(groupNum > 0, "Group size can't be less than 1")
convDesc.groups = Int(groupNum)
// initialize the convolution layer by calling the parent's (MPSCNNConvlution's) initializer
super.init(device: device,
convolutionDescriptor: convDesc,
kernelWeights: w,
biasTerms: b,
flags: MPSCNNConvolutionFlags.none)
self.destinationFeatureChannelOffset = Int(destinationFeatureChannelOffset)
// set padding for calculation of offset during encode call
padding = willPad
// unmap files at initialization of MPSCNNConvolution, the weights are copied and packed internally we no longer require these
assert(munmap(hdrW, Int(sizeWeights)) == 0, "munmap failed with errno = \(errno)")
assert(munmap(hdrB, Int(sizeBias)) == 0, "munmap failed with errno = \(errno)")
// close file descriptors
close(fd_w)
close(fd_b)
}
/**
Encode a MPSCNNKernel into a command Buffer. The operation shall proceed out-of-place.
We calculate the appropriate offset as per how TensorFlow calculates its padding using input image size and stride here.
This [Link](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/nn.py) has an explanation in header comments how tensorFlow pads its convolution input images.
- Parameters:
- commandBuffer: A valid MTLCommandBuffer to receive the encoded filter
- sourceImage: A valid MPSImage object containing the source image.
- destinationImage: A valid MPSImage to be overwritten by result image. destinationImage may not alias sourceImage
*/
override func encode(commandBuffer: MTLCommandBuffer, sourceImage: MPSImage, destinationImage: MPSImage) {
// select offset according to padding being used or not
if padding {
let pad_along_height = ((destinationImage.height - 1) * strideInPixelsY + kernelHeight - sourceImage.height)
let pad_along_width = ((destinationImage.width - 1) * strideInPixelsX + kernelWidth - sourceImage.width)
let pad_top = Int(pad_along_height / 2)
let pad_left = Int(pad_along_width / 2)
self.offset = MPSOffset(x: ((Int(kernelWidth)/2) - pad_left), y: (Int(kernelHeight/2) - pad_top), z: 0)
}
else{
self.offset = MPSOffset(x: Int(kernelWidth)/2, y: Int(kernelHeight)/2, z: 0)
}
super.encode(commandBuffer: commandBuffer, sourceImage: sourceImage, destinationImage: destinationImage)
}
}
/**
This depends on MetalPerformanceShaders.framework
The SlimMPSCNNFullyConnected is a wrapper class around MPSCNNFullyConnected used to encapsulate:
- making an MPSCNNConvolutionDescriptor,
- adding network parameters (weights and bias binaries by memory mapping the binaries)
- getting our fullyConnected layer
*/
class SlimMPSCNNFullyConnected: MPSCNNFullyConnected{
/**
Initializes a fully connected kernel.
- Parameters:
- kernelWidth: Kernel Width
- kernelHeight: Kernel Height
- inputFeatureChannels: Number feature channels in input of this layer
- outputFeatureChannels: Number feature channels from output of this layer
- neuronFilter: A neuronFilter to add at the end as activation, default is nil
- device: The MTLDevice on which this SlimMPSCNNConvolution filter will be used
- kernelParamsBinaryName: name of the layer to fetch kernelParameters by adding a prefix "weights_" or "bias_"
- destinationFeatureChannelOffset: FeatureChannel no. in the destination MPSImage to start writing from, helps with concat operations
- Returns:
A valid SlimMPSCNNFullyConnected object or nil, if failure.
*/
init(kernelWidth: UInt, kernelHeight: UInt, inputFeatureChannels: UInt, outputFeatureChannels: UInt, neuronFilter: MPSCNNNeuron? = nil, device: MTLDevice, kernelParamsBinaryName: String, destinationFeatureChannelOffset: UInt = 0){
// check whether dat file exists in Bundle or Temporary directory in the client device
let fileManager = FileManager.default
var wtPath: String?
var bsPath: String?
let wtPathBundle = Bundle.main.path(forResource: "weights_" + kernelParamsBinaryName, ofType: "dat")
let bsPathBundle = Bundle.main.path(forResource: "bias_" + kernelParamsBinaryName, ofType: "dat")
// Note: Switch from Temporary directory to Library/Caches/ if needed to persist the data after closing)
let tmpPath = URL(fileURLWithPath: NSTemporaryDirectory())
let wtPathTmp = tmpPath.appendingPathComponent("weights_" + kernelParamsBinaryName).appendingPathExtension("dat")
let bsPathTmp = tmpPath.appendingPathComponent("bias_" + kernelParamsBinaryName).appendingPathExtension("dat")
// if fileManager.fileExists(atPath:wtPathBundle!) && fileManager.fileExists(atPath:bsPathBundle!) {
if (wtPathBundle != nil) && (bsPathBundle != nil) {
// Use parameters in Bundle
wtPath = wtPathBundle
bsPath = bsPathBundle
} else if fileManager.fileExists(atPath:wtPathTmp.path) && fileManager.fileExists(atPath:bsPathTmp.path) {
// Use parameters in Tmp
wtPath = wtPathTmp.path
bsPath = bsPathTmp.path
} else {
// Read parameters from HDF5 file and store to dat file in Tmp directory
// MARK: Parse HDF5 file
guard let path = Bundle.main.path(forResource: "mnist_model_weights", ofType: "h5") else {
fatalError("Failed to get a path")
}
guard let file = File.open(path, mode: .readOnly) else {
fatalError("Failed to open file at \(path)")
}
guard let layerNamesStringAttribute = file.openStringAttribute("layer_names") else {
fatalError("Failed to open attribute 'layer_names'")
}
guard let layerNames = try? layerNamesStringAttribute.read() else {
fatalError("Failed to get layer names")
}
// guard let layerNames = file.openGroup("/")?.objectNames() else {
// fatalError("Failed to open group name '/' and failed to get layer names")
// }
// count used for file name later
var countOfConvLayer = 0
var countOfFcLayer = 0
var partOfFileName = ""
for layerName in layerNames {
guard let layerGroup = file.openGroup(layerName) else {
fatalError("Failed to open group of \(layerName)")
}
if layerGroup.objectNames().count > 0 {
// only the layer that has parameters remain
guard let wtDataset = layerGroup.openFloatDataset(layerName + "_W:0") else {
fatalError("Failed to open data set of \(layerName)_W:0")
}
guard let bsDataset = layerGroup.openFloatDataset(layerName + "_b:0") else {
fatalError("Failed to open data set of \(layerName)_b:0")
}
var dimension = wtDataset.space.dims
guard var wtArray = try? wtDataset.read() else {
fatalError("Failed to read data set of \(layerName)_W:0")
}
guard var bsArray = try? bsDataset.read() else {
fatalError("Failed to read data set of \(layerName)_b:0")
}
let wtLength = wtArray.count
let bsLength = bsArray.count
if dimension.count == 4 {
// weights for convolution layer
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 2, axis2: 3, dimensionOfArray: &dimension)
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 1, axis2: 2, dimensionOfArray: &dimension)
wtArray = SwapAxes.for4dFlatArray(originalArray: wtArray, axis1: 0, axis2: 1, dimensionOfArray: &dimension)
countOfConvLayer += 1
partOfFileName = "conv" + String(countOfConvLayer)
} else if dimension.count == 2 {
// weights for fully connected layer
wtArray = SwapAxes.for2dFlatArray(originalArray: wtArray, axis1: 0, axis2: 1, dimensionOfArray: &dimension)
countOfFcLayer += 1
partOfFileName = "fc" + String(countOfFcLayer)
} else {
fatalError("Dataset's dimension is neither 4 (convolution layer) nor 2 (fully connected layer)")
}
let wtData = NSData(bytes: &wtArray, length: wtLength * MemoryLayout<Float>.size)
let bsData = NSData(bytes: &bsArray, length: bsLength * MemoryLayout<Float>.size)
let filePath = URL(fileURLWithPath: NSTemporaryDirectory())
let wtFilePath = filePath.appendingPathComponent("weights_" + partOfFileName).appendingPathExtension("dat")
let bsFilePath = filePath.appendingPathComponent("bias_" + partOfFileName).appendingPathExtension("dat")
do {
try wtData.write(to: wtFilePath, options: .atomic)
} catch {
print(error)
}
do {
try bsData.write(to: bsFilePath, options: .atomic)
} catch {
print(error)
}
}
}
wtPath = wtPathTmp.path
bsPath = bsPathTmp.path
}
// calculate the size of weights and bias required to be memory mapped into memory
let sizeBias = outputFeatureChannels * UInt(MemoryLayout<Float>.size)
let sizeWeights = inputFeatureChannels * kernelHeight * kernelWidth * outputFeatureChannels * UInt(MemoryLayout<Float>.size)
// // get the url to this layer's weights and bias
// let wtPath = Bundle.main.path(forResource: "weights_" + kernelParamsBinaryName, ofType: "dat")
// let bsPath = Bundle.main.path(forResource: "bias_" + kernelParamsBinaryName, ofType: "dat")
// open file descriptors in read-only mode to parameter files
let fd_w = open(wtPath!, O_RDONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
let fd_b = open(bsPath!, O_RDONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)
assert(fd_w != -1, "Error: failed to open output file at \""+wtPath!+"\" errno = \(errno)\n")
assert(fd_b != -1, "Error: failed to open output file at \""+bsPath!+"\" errno = \(errno)\n")
// memory map the parameters
let hdrW = mmap(nil, Int(sizeWeights), PROT_READ, MAP_FILE | MAP_SHARED, fd_w, 0)
let hdrB = mmap(nil, Int(sizeBias), PROT_READ, MAP_FILE | MAP_SHARED, fd_b, 0)
// cast Void pointers to Float
let w = UnsafePointer<Float>(hdrW!.assumingMemoryBound(to: Float.self))
let b = UnsafePointer<Float>(hdrB!.assumingMemoryBound(to: Float.self))
assert(w != UnsafePointer<Float>(bitPattern: -1), "mmap failed with errno = \(errno)")
assert(b != UnsafePointer<Float>(bitPattern: -1), "mmap failed with errno = \(errno)")
// create appropriate convolution descriptor (in fully connected, stride is always 1)
let convDesc = MPSCNNConvolutionDescriptor(kernelWidth: Int(kernelWidth),
kernelHeight: Int(kernelHeight),
inputFeatureChannels: Int(inputFeatureChannels),
outputFeatureChannels: Int(outputFeatureChannels),
neuronFilter: neuronFilter)
// initialize the convolution layer by calling the parent's (MPSCNNFullyConnected's) initializer
super.init(device: device,
convolutionDescriptor: convDesc,
kernelWeights: w,
biasTerms: b,
flags: MPSCNNConvolutionFlags.none)
self.destinationFeatureChannelOffset = Int(destinationFeatureChannelOffset)
// unmap files at initialization of MPSCNNFullyConnected, the weights are copied and packed internally we no longer require these
assert(munmap(hdrW, Int(sizeWeights)) == 0, "munmap failed with errno = \(errno)")
assert(munmap(hdrB, Int(sizeBias)) == 0, "munmap failed with errno = \(errno)")
// close file descriptors
close(fd_w)
close(fd_b)
}
}