Skip to content

[Swift] Rename CotangentVector to TangentVector. #144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dan-zheng merged 1 commit into from
Jun 26, 2019
Merged

[Swift] Rename CotangentVector to TangentVector. #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dev_swift/03_minibatch_training.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -779,7 +779,7 @@
" lossFunc: @escaping @differentiable (Opt.Model.Output, @nondiff Label) -> Tensor<Opt.Scalar>\n",
") where Opt.Model: Layer,\n",
" Opt.Model.Input: TensorGroup,\n",
" Opt.Model.CotangentVector == Opt.Model.AllDifferentiableVariables,\n",
" Opt.Model.TangentVector == Opt.Model.AllDifferentiableVariables,\n",
" Opt.Scalar: TensorFlowFloatingPoint\n",
"{\n",
" for batch in ds {\n",
Expand Down
10 changes: 5 additions & 5 deletions dev_swift/07_batchnorm.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,8 @@
"\n",
" @differentiating(forward)\n",
" func gradForward(_ input: Input) ->\n",
" (value: Output, pullback: (Self.Output.CotangentVector) ->\n",
" (Self.CotangentVector, Self.Input.CotangentVector)) {\n",
" (value: Output, pullback: (Self.Output.TangentVector) ->\n",
" (Self.TangentVector, Self.Input.TangentVector)) {\n",
" switch Context.local.learningPhase {\n",
" case .training:\n",
" return valueWithPullback(at: input) { $0.forwardTraining ($1) }\n",
Expand Down Expand Up @@ -382,9 +382,9 @@
" static func _vjpFusedBatchNorm(\n",
" _ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar\n",
" ) -> (BatchNormResult<Scalar>, \n",
" (BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector, \n",
" Tensor<Scalar>.CotangentVector, \n",
" Tensor<Scalar>.CotangentVector)) {\n",
" (BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector, \n",
" Tensor<Scalar>.TangentVector, \n",
" Tensor<Scalar>.TangentVector)) {\n",
" let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)\n",
" \n",
" return (\n",
Expand Down
12 changes: 6 additions & 6 deletions dev_swift/07b_batchnorm_lesson.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,8 @@
"\n",
" @differentiating(forward)\n",
" func gradForward(_ input: Input) ->\n",
" (value: Output, pullback: (Self.Output.CotangentVector) ->\n",
" (Self.CotangentVector, Self.Input.CotangentVector)) {\n",
" (value: Output, pullback: (Self.Output.TangentVector) ->\n",
" (Self.TangentVector, Self.Input.TangentVector)) {\n",
" switch Context.local.learningPhase {\n",
" case .training: return valueWithPullback(at: input) { $0.forwardTraining($1) }\n",
" case .inference: return valueWithPullback(at: input) { $0.forwardInference($1) }\n",
Expand Down Expand Up @@ -260,7 +260,7 @@
"source": [
"//export\n",
"public struct ConvNorm<NormType: Norm & FALayer>: FALayer\n",
" where NormType.AllDifferentiableVariables == NormType.CotangentVector {\n",
" where NormType.AllDifferentiableVariables == NormType.TangentVector {\n",
" public var conv: FANoBiasConv2D<Float>\n",
" public var norm: NormType\n",
" \n",
Expand All @@ -285,7 +285,7 @@
"source": [
"//export\n",
"public struct CnnModelNormed<NormType: Norm & FALayer>: FALayer\n",
" where NormType.AllDifferentiableVariables == NormType.CotangentVector {\n",
" where NormType.AllDifferentiableVariables == NormType.TangentVector {\n",
" public var convs: [ConvNorm<NormType>]\n",
" public var pool = FAGlobalAvgPool2D<Float>()\n",
" public var linear: FADense<Float>\n",
Expand Down Expand Up @@ -416,10 +416,10 @@
"\n",
"func xlaCompiled<T : Differentiable & TensorGroup, U : Differentiable & TensorGroup>(\n",
" _ fn: @escaping @differentiable (T) -> U) -> CompiledFunction<T, U>\n",
" where T.CotangentVector : TensorGroup, U.CotangentVector : TensorGroup {\n",
" where T.TangentVector : TensorGroup, U.TangentVector : TensorGroup {\n",
" let xlaCompiledFn: (T) -> U = _graph(fn, useXLA: true)\n",
" let xlaCompiledPullback = _graph(\n",
" { (pbArgs: PullbackArgs<T, U.CotangentVector>) in\n",
" { (pbArgs: PullbackArgs<T, U.TangentVector>) in\n",
" pullback(at: pbArgs.input, in: fn)(pbArgs.cotangent) },\n",
" useXLA: true\n",
" )\n",
Expand Down
8 changes: 4 additions & 4 deletions dev_swift/09_optimizer.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@
"source": [
"//export\n",
"public class StatefulOptimizer<Model: Layer>\n",
" where Model.AllDifferentiableVariables == Model.CotangentVector {\n",
" where Model.AllDifferentiableVariables == Model.TangentVector {\n",
" public typealias ModelKeyPath = WritableKeyPath<Model.AllDifferentiableVariables, TF>\n",
" public typealias SplitDict = [ModelKeyPath: Int]\n",
" public var hpGroups: [[String:Float]]\n",
Expand All @@ -291,7 +291,7 @@
" \n",
" public func update(\n",
" _ variables: inout Model.AllDifferentiableVariables,\n",
" along direction: Model.CotangentVector\n",
" along direction: Model.TangentVector\n",
" ) {\n",
" for kp in variables.keyPaths {\n",
" var 𝛁p = direction[keyPath: kp]\n",
Expand Down Expand Up @@ -949,7 +949,7 @@
"source": [
"// export\n",
"extension Learner where Opt.Scalar: BinaryFloatingPoint, \n",
" Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{\n",
" Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{\n",
" public class ParamScheduler: Delegate {\n",
" public override var order: Int { return 1 }\n",
" public typealias ScheduleFunc = (Float) -> Float\n",
Expand Down Expand Up @@ -1011,7 +1011,7 @@
"source": [
"// export\n",
"extension Learner where Opt.Scalar: BinaryFloatingPoint, \n",
" Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{\n",
" Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{\n",
"\n",
" public func addOneCycleDelegates(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5, \n",
" moms: (Float,Float,Float) = (0.95,0.85,0.95)) {\n",
Expand Down
6 changes: 3 additions & 3 deletions dev_swift/11_imagenette.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,12 +312,12 @@
" @differentiating(callAsFunction)\n",
" func gradForward(_ input: Input) ->\n",
" (value: Input,\n",
" pullback: (Self.Input.CotangentVector) ->\n",
" (Self.CotangentVector, Self.Input.CotangentVector)) {\n",
" pullback: (Self.Input.TangentVector) ->\n",
" (Self.TangentVector, Self.Input.TangentVector)) {\n",
" if isOn {\n",
" return valueWithPullback(at: input) { $0.forward($1) } \n",
" } else {\n",
" return (input, { (Self.CotangentVector.zero, $0) }) \n",
" return (input, { (Self.TangentVector.zero, $0) }) \n",
" }\n",
" }\n",
"}"
Expand Down
10 changes: 5 additions & 5 deletions dev_swift/FastaiNotebook_07_batchnorm/Sources/FastaiNotebook_07_batchnorm/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
10 changes: 5 additions & 5 deletions ...wift/FastaiNotebook_08_data_block/Sources/FastaiNotebook_08_data_block/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
10 changes: 5 additions & 5 deletions ...geneous_dictionary/Sources/FastaiNotebook_08a_heterogeneous_dictionary/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
10 changes: 5 additions & 5 deletions ...k_08c_data_block_generic/Sources/FastaiNotebook_08c_data_block_generic/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
10 changes: 5 additions & 5 deletions dev_swift/FastaiNotebook_09_optimizer/Sources/FastaiNotebook_09_optimizer/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
8 changes: 4 additions & 4 deletions dev_swift/FastaiNotebook_09_optimizer/Sources/FastaiNotebook_09_optimizer/09_optimizer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public func initState<Model: Layer>(for model: Model, names: [String])
}

public class StatefulOptimizer<Model: Layer>
where Model.AllDifferentiableVariables == Model.CotangentVector {
where Model.AllDifferentiableVariables == Model.TangentVector {
public typealias ModelKeyPath = WritableKeyPath<Model.AllDifferentiableVariables, TF>
public typealias SplitDict = [ModelKeyPath: Int]
public var hpGroups: [[String:Float]]
Expand All @@ -85,7 +85,7 @@ public class StatefulOptimizer<Model: Layer>

public func update(
_ variables: inout Model.AllDifferentiableVariables,
along direction: Model.CotangentVector
along direction: Model.TangentVector
) {
for kp in variables.keyPaths {
var 𝛁p = direction[keyPath: kp]
Expand Down Expand Up @@ -273,7 +273,7 @@ public extension StatefulOptimizer {
}

extension Learner where Opt.Scalar: BinaryFloatingPoint,
Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{
Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{
public class ParamScheduler: Delegate {
public override var order: Int { return 1 }
public typealias ScheduleFunc = (Float) -> Float
Expand Down Expand Up @@ -312,7 +312,7 @@ public func oneCycleSchedulers(_ lrMax: Float, pctStart:Float=0.25, divStart: Fl
}

extension Learner where Opt.Scalar: BinaryFloatingPoint,
Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{
Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{

public func addOneCycleDelegates(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5,
moms: (Float,Float,Float) = (0.95,0.85,0.95)) {
Expand Down
10 changes: 5 additions & 5 deletions dev_swift/FastaiNotebook_10_mixup_ls/Sources/FastaiNotebook_10_mixup_ls/07_batchnorm.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ extension LearningPhaseDependent {

@differentiating(forward)
func gradForward(_ input: Input) ->
(value: Output, pullback: (Self.Output.CotangentVector) ->
(Self.CotangentVector, Self.Input.CotangentVector)) {
(value: Output, pullback: (Self.Output.TangentVector) ->
(Self.TangentVector, Self.Input.TangentVector)) {
switch Context.local.learningPhase {
case .training:
return valueWithPullback(at: input) { $0.forwardTraining ($1) }
Expand Down Expand Up @@ -154,9 +154,9 @@ public struct TFBatchNorm<Scalar: TensorFlowFloatingPoint>: LearningPhaseDepende
static func _vjpFusedBatchNorm(
_ x : Tensor<Scalar>, scale: Tensor<Scalar>, offset: Tensor<Scalar>, epsilon: Scalar
) -> (BatchNormResult<Scalar>,
(BatchNormResult<Scalar>.CotangentVector) -> (Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector,
Tensor<Scalar>.CotangentVector)) {
(BatchNormResult<Scalar>.TangentVector) -> (Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector,
Tensor<Scalar>.TangentVector)) {
let bnresult = fusedBatchNorm(x, scale: scale, offset: offset, epsilon: epsilon)

return (
Expand Down
8 changes: 4 additions & 4 deletions dev_swift/FastaiNotebook_10_mixup_ls/Sources/FastaiNotebook_10_mixup_ls/09_optimizer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public func initState<Model: Layer>(for model: Model, names: [String])
}

public class StatefulOptimizer<Model: Layer>
where Model.AllDifferentiableVariables == Model.CotangentVector {
where Model.AllDifferentiableVariables == Model.TangentVector {
public typealias ModelKeyPath = WritableKeyPath<Model.AllDifferentiableVariables, TF>
public typealias SplitDict = [ModelKeyPath: Int]
public var hpGroups: [[String:Float]]
Expand All @@ -85,7 +85,7 @@ public class StatefulOptimizer<Model: Layer>

public func update(
_ variables: inout Model.AllDifferentiableVariables,
along direction: Model.CotangentVector
along direction: Model.TangentVector
) {
for kp in variables.keyPaths {
var 𝛁p = direction[keyPath: kp]
Expand Down Expand Up @@ -273,7 +273,7 @@ public extension StatefulOptimizer {
}

extension Learner where Opt.Scalar: BinaryFloatingPoint,
Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{
Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{
public class ParamScheduler: Delegate {
public override var order: Int { return 1 }
public typealias ScheduleFunc = (Float) -> Float
Expand Down Expand Up @@ -312,7 +312,7 @@ public func oneCycleSchedulers(_ lrMax: Float, pctStart:Float=0.25, divStart: Fl
}

extension Learner where Opt.Scalar: BinaryFloatingPoint,
Opt.Model.AllDifferentiableVariables == Opt.Model.CotangentVector{
Opt.Model.AllDifferentiableVariables == Opt.Model.TangentVector{

public func addOneCycleDelegates(_ lrMax: Float, pctStart:Float=0.25, divStart: Float = 10, divEnd: Float = 1e5,
moms: (Float,Float,Float) = (0.95,0.85,0.95)) {
Expand Down
Loading