From 158a0b3f20f7cb4f295e20f20f9a74b54b4dcd64 Mon Sep 17 00:00:00 2001 From: CompatHelper Julia Date: Wed, 13 Aug 2025 00:06:27 +0000 Subject: [PATCH 01/12] CompatHelper: bump compat for JET in [weakdeps] to 0.10, (keep existing compat) --- Project.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index b6ef92c851..e157394a6a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Mooncake" uuid = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" authors = ["Will Tebbutt, Hong Ge, and contributors"] -version = "0.4.148" +version = "0.4.149" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" @@ -61,20 +61,20 @@ FunctionWrappers = "1.1.3" GPUArraysCore = "0.1, 0.2" Graphs = "1" InteractiveUtils = "1" -JET = "0.9" +JET = "0.9, 0.10" JuliaFormatter = "1.0, 2.1" JuliaInterpreter = "0.9" LinearAlgebra = "1" LuxLib = "1" -MistyClosures = "2" MLDataDevices = "1.10.0" +MistyClosures = "2" NNlib = "0.9" Pkg = "1" Random = "1" SLEEFPirates = "0.6.43" SpecialFunctions = "2" -Static = "1.1.1" StableRNGs = "1" +Static = "1.1.1" Test = "1" julia = "~1.10.8, 1.11.6" From 83889dc6718fb27206709ea8ff06d988291f51a9 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:33:55 +0100 Subject: [PATCH 02/12] Set tangent_type for AbstractLogger to NoTangent (#710) * Initial plan * Set tangent_type for Logging.ConsoleLogger to NoTangent Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Add ConsoleLogger test to verify AbstractLogger tangent type coverage Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Add Base.CoreLogging.SimpleLogger to type union Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Update test/tangents.jl Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --- src/tangents.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tangents.jl b/src/tangents.jl index 73a40371b6..5f6c26be51 100644 --- a/src/tangents.jl +++ b/src/tangents.jl @@ -370,6 +370,8 @@ tangent_type(::Type{<:Base.TTY}) = NoTangent tangent_type(::Type{<:IOStream}) = NoTangent +tangent_type(::Type{<:Base.CoreLogging.AbstractLogger}) = NoTangent + function split_union_tuple_type(tangent_types) # Create first split. @@ -1353,6 +1355,7 @@ tangents, but they're unable to check that `increment!!` is correct in an absolu (Union, NoTangent), (UnionAll, NoTangent), (typeof(<:), NoTangent), + (Base.CoreLogging.SimpleLogger, NoTangent), (IOStream(""), NoTangent), ] # Construct test cases containing circular references. These typically require multiple From 93dd893b6c171420dd741dc1f20f5fe8a067bb7d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:34:41 +0100 Subject: [PATCH 03/12] Add better error messages for tangent accessors when types are passed (#707) * Initial plan * Add better error messages for tangent accessors when types are passed Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Hong Ge Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --- src/fwds_rvs_data.jl | 16 ++++++++++++++++ test/fwds_rvs_data.jl | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/fwds_rvs_data.jl b/src/fwds_rvs_data.jl index 0ad4981e3e..90b4bc3c35 100644 --- a/src/fwds_rvs_data.jl +++ b/src/fwds_rvs_data.jl @@ -258,6 +258,14 @@ function fdata(t::T) where {T} return F(fdata(t.fields)) end +function fdata(::Type{T}) where {T} + throw( + error( + "$T is a type. Perhaps you meant fdata_type($T) or fdata(instance_of_tangent)?" + ), + ) +end + function fdata(t::T) where {T<:PossiblyUninitTangent} F = fdata_type(T) return is_init(t) ? F(fdata(val(t))) : F() @@ -530,6 +538,14 @@ function rdata(t::T) where {T} return R(rdata(t.fields)) end +function rdata(::Type{T}) where {T} + throw( + error( + "$T is a type. Perhaps you meant rdata_type($T) or rdata(instance_of_tangent)?" + ), + ) +end + function rdata(t::T) where {T<:PossiblyUninitTangent} R = rdata_type(T) return is_init(t) ? R(rdata(val(t))) : R() diff --git a/test/fwds_rvs_data.jl b/test/fwds_rvs_data.jl index ebe485ac1c..7f17aa7c82 100644 --- a/test/fwds_rvs_data.jl +++ b/test/fwds_rvs_data.jl @@ -157,4 +157,41 @@ end @test verify_rdata_value(Ptr{Float64}(), NoRData()) === nothing end end + + @testset "Error messages for types passed to accessor functions" begin + # Test that passing types to fdata throws helpful error message + @test_throws "Float64 is a type. Perhaps you meant fdata_type(Float64) or fdata(instance_of_tangent)?" fdata( + Float64 + ) + @test_throws "Int64 is a type. Perhaps you meant fdata_type(Int64) or fdata(instance_of_tangent)?" fdata( + Int64 + ) + + # Test that passing types to rdata throws helpful error message + @test_throws "Float64 is a type. Perhaps you meant rdata_type(Float64) or rdata(instance_of_tangent)?" rdata( + Float64 + ) + @test_throws "Int64 is a type. Perhaps you meant rdata_type(Int64) or rdata(instance_of_tangent)?" rdata( + Int64 + ) + + # Test with more complex types - using exact string matching since the message is deterministic + @test_throws "Vector{Float64} is a type. Perhaps you meant fdata_type(Vector{Float64}) or fdata(instance_of_tangent)?" fdata( + Vector{Float64} + ) + @test_throws "Vector{Float64} is a type. Perhaps you meant rdata_type(Vector{Float64}) or rdata(instance_of_tangent)?" rdata( + Vector{Float64} + ) + + # Verify that existing functionality with instances still works + x = 5.0 + t = zero_tangent(x) + @test fdata(t) isa NoFData + @test rdata(t) isa Float64 + + arr = [1.0, 2.0] + t_arr = zero_tangent(arr) + @test fdata(t_arr) isa Vector{Float64} + @test rdata(t_arr) isa NoRData + end end From 5f2efda360dce242b34e8e0436b623569240d05d Mon Sep 17 00:00:00 2001 From: Will Tebbutt <3628294+willtebbutt@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:37:51 +0100 Subject: [PATCH 04/12] Tidy up (#715) * Tidy up error-generating methods * Remove redundant tests * Remove more redundant test cases * Bump patch version --- Project.toml | 2 +- src/fwds_rvs_data.jl | 12 ++---------- test/fwds_rvs_data.jl | 38 +++----------------------------------- 3 files changed, 6 insertions(+), 46 deletions(-) diff --git a/Project.toml b/Project.toml index 3837aef17d..2b806a8950 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Mooncake" uuid = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" authors = ["Will Tebbutt, Hong Ge, and contributors"] -version = "0.4.152" +version = "0.4.153" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" diff --git a/src/fwds_rvs_data.jl b/src/fwds_rvs_data.jl index 90b4bc3c35..2cf7bceee6 100644 --- a/src/fwds_rvs_data.jl +++ b/src/fwds_rvs_data.jl @@ -259,11 +259,7 @@ function fdata(t::T) where {T} end function fdata(::Type{T}) where {T} - throw( - error( - "$T is a type. Perhaps you meant fdata_type($T) or fdata(instance_of_tangent)?" - ), - ) + error("$T is a type. Perhaps you meant fdata_type($T) or fdata(instance_of_tangent)?") end function fdata(t::T) where {T<:PossiblyUninitTangent} @@ -539,11 +535,7 @@ function rdata(t::T) where {T} end function rdata(::Type{T}) where {T} - throw( - error( - "$T is a type. Perhaps you meant rdata_type($T) or rdata(instance_of_tangent)?" - ), - ) + error("$T is a type. Perhaps you meant rdata_type($T) or rdata(instance_of_tangent)?") end function rdata(t::T) where {T<:PossiblyUninitTangent} diff --git a/test/fwds_rvs_data.jl b/test/fwds_rvs_data.jl index 7f17aa7c82..63f775caf9 100644 --- a/test/fwds_rvs_data.jl +++ b/test/fwds_rvs_data.jl @@ -158,40 +158,8 @@ end end end - @testset "Error messages for types passed to accessor functions" begin - # Test that passing types to fdata throws helpful error message - @test_throws "Float64 is a type. Perhaps you meant fdata_type(Float64) or fdata(instance_of_tangent)?" fdata( - Float64 - ) - @test_throws "Int64 is a type. Perhaps you meant fdata_type(Int64) or fdata(instance_of_tangent)?" fdata( - Int64 - ) - - # Test that passing types to rdata throws helpful error message - @test_throws "Float64 is a type. Perhaps you meant rdata_type(Float64) or rdata(instance_of_tangent)?" rdata( - Float64 - ) - @test_throws "Int64 is a type. Perhaps you meant rdata_type(Int64) or rdata(instance_of_tangent)?" rdata( - Int64 - ) - - # Test with more complex types - using exact string matching since the message is deterministic - @test_throws "Vector{Float64} is a type. Perhaps you meant fdata_type(Vector{Float64}) or fdata(instance_of_tangent)?" fdata( - Vector{Float64} - ) - @test_throws "Vector{Float64} is a type. Perhaps you meant rdata_type(Vector{Float64}) or rdata(instance_of_tangent)?" rdata( - Vector{Float64} - ) - - # Verify that existing functionality with instances still works - x = 5.0 - t = zero_tangent(x) - @test fdata(t) isa NoFData - @test rdata(t) isa Float64 - - arr = [1.0, 2.0] - t_arr = zero_tangent(arr) - @test fdata(t_arr) isa Vector{Float64} - @test rdata(t_arr) isa NoRData + @testset "Helpful error messages for misuse of fdata and rdata" begin + @test_throws "Float64 is a type. Perhaps you meant" fdata(Float64) + @test_throws "Float64 is a type. Perhaps you meant" rdata(Float64) end end From 4057e0261aabe68a193dd4401f35167cf68c9df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateus=20Ara=C3=BAjo?= Date: Wed, 27 Aug 2025 19:23:48 +0200 Subject: [PATCH 05/12] add rule for complex logdet (#664) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * working complex logdet * bump version * proper getrf! tests * improve tests * move tests * generalize tests * formatting and version --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> Signed-off-by: Mateus Araújo Co-authored-by: Hong Ge <3279477+yebai@users.noreply.github.com> --- Project.toml | 2 +- src/rrules/blas.jl | 7 +++++++ src/rrules/lapack.jl | 49 ++++++++++++++++++++++++++++++-------------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/Project.toml b/Project.toml index 2b806a8950..7fcc4071f6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Mooncake" uuid = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" authors = ["Will Tebbutt, Hong Ge, and contributors"] -version = "0.4.153" +version = "0.4.154" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" diff --git a/src/rrules/blas.jl b/src/rrules/blas.jl index f7555226d5..f7fd44a9e5 100644 --- a/src/rrules/blas.jl +++ b/src/rrules/blas.jl @@ -1109,6 +1109,13 @@ function invertible_blas_matrices(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int end end +function positive_definite_blas_matrices(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int) + return map(blas_matrices(rng, P, p, p)) do A + A .= A'A + I + return A + end +end + function blas_vectors(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int) xs = Any[ randn(rng, P, p), diff --git a/src/rrules/lapack.jl b/src/rrules/lapack.jl index ede01c2217..2d5229e21c 100644 --- a/src/rrules/lapack.jl +++ b/src/rrules/lapack.jl @@ -1,14 +1,14 @@ # See https://sethaxen.com/blog/2021/02/differentiating-the-lu-decomposition/ for details. -@is_primitive(MinimalCtx, Tuple{typeof(LAPACK.getrf!),AbstractMatrix{<:BlasRealFloat}}) +@is_primitive(MinimalCtx, Tuple{typeof(LAPACK.getrf!),AbstractMatrix{<:BlasFloat}}) function frule!!( ::Dual{typeof(LAPACK.getrf!)}, A_dA::Dual{<:AbstractMatrix{P}} -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} _, ipiv, info = LAPACK.getrf!(primal(A_dA)) return _getrf_fwd(A_dA, ipiv, info) end function rrule!!( ::CoDual{typeof(LAPACK.getrf!)}, _A::CoDual{<:AbstractMatrix{P}} -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} A, dA = arrayify(_A) A_copy = copy(A) @@ -28,16 +28,14 @@ end @is_primitive( MinimalCtx, - Tuple{ - typeof(Core.kwcall),NamedTuple,typeof(LAPACK.getrf!),AbstractMatrix{<:BlasRealFloat} - }, + Tuple{typeof(Core.kwcall),NamedTuple,typeof(LAPACK.getrf!),AbstractMatrix{<:BlasFloat}}, ) function frule!!( ::Dual{typeof(Core.kwcall)}, _kwargs::Dual{<:NamedTuple}, ::Dual{typeof(getrf!)}, A_dA::Dual{<:AbstractMatrix{P}}, -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} check = primal(_kwargs).check _, ipiv, info = LAPACK.getrf!(primal(A_dA); check) return _getrf_fwd(A_dA, ipiv, info) @@ -47,7 +45,7 @@ function rrule!!( _kwargs::CoDual{<:NamedTuple}, ::CoDual{typeof(getrf!)}, _A::CoDual{<:AbstractMatrix{P}}, -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} check = _kwargs.x.check A, dA = arrayify(_A) A_copy = copy(A) @@ -505,6 +503,7 @@ end function generate_hand_written_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) rng = rng_ctor(123) Ps = [Float64, Float32] + complexPs = [Float64, Float32, ComplexF64, ComplexF32] bools = [false, true] test_cases = vcat( @@ -516,7 +515,7 @@ function generate_hand_written_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) (false, :stability, nothing, getrf!, A) end end..., - map_prod(bools, Ps) do (check, P) + map_prod(bools, complexPs) do (check, P) As = blas_matrices(rng, P, 5, 5) ipiv = Vector{Int}(undef, 5) return map(As) do A @@ -583,13 +582,33 @@ end function generate_derived_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) rng = rng_ctor(123) + complexPs = [Float64, Float32, ComplexF64, ComplexF32] getrf_wrapper!(x, check) = getrf!(x; check) - test_cases = vcat(map_prod([false, true], [Float64, Float32]) do (check, P) - As = blas_matrices(rng, P, 5, 5) - return map(As) do A - (false, :none, nothing, getrf_wrapper!, A, check) - end - end...) + test_cases = vcat( + # getrf + map_prod([false, true], complexPs) do (check, P) + As = blas_matrices(rng, P, 5, 5) + return map(As) do A + (false, :none, nothing, getrf_wrapper!, A, check) + end + end..., + + # real logdet + map([Float64, Float32]) do P + As = positive_definite_blas_matrices(rng, P, 3) + return map(As) do A + (false, :none, nothing, logdet, A) + end + end..., + + # complex logdet + map(complexPs) do P + As = blas_matrices(rng, P, 3, 3) + return map(As) do A + (false, :none, nothing, real ∘ logdet ∘ complex, A) + end + end..., + ) memory = Any[] return test_cases, memory end From ff1dbd4b969fa0599d6aba84270557f36706600a Mon Sep 17 00:00:00 2001 From: Hong Ge <3279477+yebai@users.noreply.github.com> Date: Wed, 27 Aug 2025 22:06:15 +0100 Subject: [PATCH 06/12] Delete .github/copilot-instructions.md (#722) Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --- .github/copilot-instructions.md | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md deleted file mode 100644 index 6111b7439c..0000000000 --- a/.github/copilot-instructions.md +++ /dev/null @@ -1,14 +0,0 @@ -This repo implements an automatic differentiation package for the Julia language. - -Please follow these guidelines when contributing: - -## Code Standards - -- Never commit changes to `Project.toml` to PRs. -- Format modified files using `JuliaFormatter`, eg, `using JuliaFormatter; format(".")` from the root of the repo. - -## Key Guidelines -- Follow Go best practices and idiomatic patterns -- Maintain existing code structure and organisation -- Write unit tests for new functionality. -- Document public APIs and complex logic. Suggest changes to the `docs/` folder when appropriate From 67242d9857455dd9de21d1f093344389b90eb2a4 Mon Sep 17 00:00:00 2001 From: Hong Ge <3279477+yebai@users.noreply.github.com> Date: Thu, 28 Aug 2025 00:07:32 +0100 Subject: [PATCH 07/12] Uncomment b_binv_test_case for PlanarLayer (#727) Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --- test/integration_testing/bijectors/bijectors.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration_testing/bijectors/bijectors.jl b/test/integration_testing/bijectors/bijectors.jl index 998c1f34d0..289f97c6d5 100644 --- a/test/integration_testing/bijectors/bijectors.jl +++ b/test/integration_testing/bijectors/bijectors.jl @@ -50,7 +50,7 @@ end 1 0 0 0 0 1 ]), (3, 3)), - # b_binv_test_case(Bijectors.PlanarLayer(3), (3, 3)), + b_binv_test_case(Bijectors.PlanarLayer(3), (3, 3)), b_binv_test_case(Bijectors.RadialLayer(3), 3), b_binv_test_case(Bijectors.Reshape((2, 3), (3, 2)), (2, 3)), b_binv_test_case(Bijectors.Scale(0.2), 3), From ec47ca84249a9577bf89412c744d311ff60bd6f7 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 Aug 2025 00:53:05 +0100 Subject: [PATCH 08/12] Add DiffEq integration test for Mooncake (#726) * Initial plan * Add DiffEq integration test with original example from issue Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Finalize DiffEq integration test with comprehensive structure Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Remove @info and @test_broken statements to reduce clutter Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --- .github/workflows/CI.yml | 1 + test/integration_testing/diffeq/Project.toml | 6 +++ test/integration_testing/diffeq/diffeq.jl | 53 ++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 test/integration_testing/diffeq/Project.toml create mode 100644 test/integration_testing/diffeq/diffeq.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9d4e266024..a89e25f686 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -95,6 +95,7 @@ jobs: {test_type: 'integration_testing', label: 'array'}, {test_type: 'integration_testing', label: 'bijectors'}, {test_type: 'integration_testing', label: 'diff_tests'}, + {test_type: 'integration_testing', label: 'diffeq'}, {test_type: 'integration_testing', label: 'dispatch_doctor'}, {test_type: 'integration_testing', label: 'distributions'}, {test_type: 'integration_testing', label: 'dynamicppl'}, diff --git a/test/integration_testing/diffeq/Project.toml b/test/integration_testing/diffeq/Project.toml new file mode 100644 index 0000000000..98e015ff48 --- /dev/null +++ b/test/integration_testing/diffeq/Project.toml @@ -0,0 +1,6 @@ +[deps] +Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" +StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" \ No newline at end of file diff --git a/test/integration_testing/diffeq/diffeq.jl b/test/integration_testing/diffeq/diffeq.jl new file mode 100644 index 0000000000..00ae9e1081 --- /dev/null +++ b/test/integration_testing/diffeq/diffeq.jl @@ -0,0 +1,53 @@ +using Pkg +Pkg.activate(@__DIR__) +Pkg.develop(; path=joinpath(@__DIR__, "..", "..", "..")) + +using OrdinaryDiffEq, SciMLSensitivity, Mooncake, StableRNGs, Test +using Mooncake.TestUtils: test_rule + +# Helper function for Mooncake gradient computation +mooncake_gradient(f, x) = Mooncake.value_and_gradient!!(Mooncake.build_rrule(f, x), f, x)[2][2] + +# Define the ODE function from the original issue +odef(du, u, p, t) = du .= u .* p + +# Define the sensitivity loss function from the original issue +struct senseloss0{T} + sense::T +end + +function (f::senseloss0)(u0p) + prob = ODEProblem{true}(odef, u0p[1:1], (0.0, 1.0), u0p[2:2]) + sum(solve(prob, Tsit5(), abstol = 1e-12, reltol = 1e-12, saveat = 0.1)) +end + +@testset "diffeq" begin + rng = StableRNG(123456) + + # Test parameters from the original issue + u0p = [2.0, 3.0] + + @testset "senseloss0 with InterpolatingAdjoint" begin + sense_func = senseloss0(InterpolatingAdjoint()) + + # First test that the function works + @testset "Function evaluation" begin + result = sense_func(u0p) + @test result isa Real + @test isfinite(result) + end + + # Test Mooncake gradient computation + @testset "mooncake_gradient computation" begin + dup_mc = mooncake_gradient(sense_func, u0p) + @test dup_mc isa Vector + @test length(dup_mc) == 2 + @test all(isfinite, dup_mc) + end + + # Test with Mooncake's test_rule + @testset "test_rule evaluation" begin + test_rule(rng, sense_func, u0p; is_primitive=false, unsafe_perturb=true) + end + end +end \ No newline at end of file From ea19541c5c150fbd1f5dc3d90e7ea89388c7b1ae Mon Sep 17 00:00:00 2001 From: Hong Ge <3279477+yebai@users.noreply.github.com> Date: Thu, 28 Aug 2025 01:27:26 +0100 Subject: [PATCH 09/12] fix formatting (#728) --- test/integration_testing/diffeq/diffeq.jl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/integration_testing/diffeq/diffeq.jl b/test/integration_testing/diffeq/diffeq.jl index 00ae9e1081..e76678c479 100644 --- a/test/integration_testing/diffeq/diffeq.jl +++ b/test/integration_testing/diffeq/diffeq.jl @@ -6,7 +6,9 @@ using OrdinaryDiffEq, SciMLSensitivity, Mooncake, StableRNGs, Test using Mooncake.TestUtils: test_rule # Helper function for Mooncake gradient computation -mooncake_gradient(f, x) = Mooncake.value_and_gradient!!(Mooncake.build_rrule(f, x), f, x)[2][2] +function mooncake_gradient(f, x) + Mooncake.value_and_gradient!!(Mooncake.build_rrule(f, x), f, x)[2][2] +end # Define the ODE function from the original issue odef(du, u, p, t) = du .= u .* p @@ -18,25 +20,25 @@ end function (f::senseloss0)(u0p) prob = ODEProblem{true}(odef, u0p[1:1], (0.0, 1.0), u0p[2:2]) - sum(solve(prob, Tsit5(), abstol = 1e-12, reltol = 1e-12, saveat = 0.1)) + sum(solve(prob, Tsit5(); abstol=1e-12, reltol=1e-12, saveat=0.1)) end @testset "diffeq" begin rng = StableRNG(123456) - + # Test parameters from the original issue u0p = [2.0, 3.0] - + @testset "senseloss0 with InterpolatingAdjoint" begin sense_func = senseloss0(InterpolatingAdjoint()) - + # First test that the function works @testset "Function evaluation" begin result = sense_func(u0p) @test result isa Real @test isfinite(result) end - + # Test Mooncake gradient computation @testset "mooncake_gradient computation" begin dup_mc = mooncake_gradient(sense_func, u0p) @@ -44,10 +46,10 @@ end @test length(dup_mc) == 2 @test all(isfinite, dup_mc) end - + # Test with Mooncake's test_rule @testset "test_rule evaluation" begin test_rule(rng, sense_func, u0p; is_primitive=false, unsafe_perturb=true) end end -end \ No newline at end of file +end From a3e79bd1464a5a0d4e8c44b4cc583f278a38124c Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 Aug 2025 11:32:08 +0100 Subject: [PATCH 10/12] Fix DiffEq integration test by restricting to reverse mode only (#731) * Initial plan * Fix DiffEq integration test by testing reverse mode only Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * formatting --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> Co-authored-by: Hong Ge --- test/integration_testing/diffeq/diffeq.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration_testing/diffeq/diffeq.jl b/test/integration_testing/diffeq/diffeq.jl index e76678c479..b998d5fe41 100644 --- a/test/integration_testing/diffeq/diffeq.jl +++ b/test/integration_testing/diffeq/diffeq.jl @@ -4,6 +4,7 @@ Pkg.develop(; path=joinpath(@__DIR__, "..", "..", "..")) using OrdinaryDiffEq, SciMLSensitivity, Mooncake, StableRNGs, Test using Mooncake.TestUtils: test_rule +using Mooncake: ReverseMode # Helper function for Mooncake gradient computation function mooncake_gradient(f, x) @@ -49,7 +50,14 @@ end # Test with Mooncake's test_rule @testset "test_rule evaluation" begin - test_rule(rng, sense_func, u0p; is_primitive=false, unsafe_perturb=true) + test_rule( + rng, + sense_func, + u0p; + is_primitive=false, + unsafe_perturb=true, + mode=ReverseMode, + ) end end end From 315911aef6e98591d1a4f6b145de1dfaf3bdf098 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 Aug 2025 11:36:35 +0100 Subject: [PATCH 11/12] Fix IR docs duplication and improve documentation structure (#718) * Initial plan * Update IR docs title and remove duplication, add comparison section Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Address PR feedback: update title and simplify Optics reference Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Restore IRCode field details that were incorrectly removed Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --- docs/src/developer_documentation/ir_representation.md | 8 +++++++- docs/src/developer_documentation/reverse_mode_design.md | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/src/developer_documentation/ir_representation.md b/docs/src/developer_documentation/ir_representation.md index d9d03c469a..367c20819d 100644 --- a/docs/src/developer_documentation/ir_representation.md +++ b/docs/src/developer_documentation/ir_representation.md @@ -1,4 +1,4 @@ -# IR Representation +# IR Representations and Code Transformations Mooncake.jl works by transforming Julia's SSA-form (static single assignment) Intermediate Representation (IR), so a good understanding of Julia's IR is needed to understand Mooncake. Furthermore, Mooncake holds Julia's IR in a different data structure than the one usually used when producing code for reverse-mode AD. @@ -555,6 +555,12 @@ Reverse-mode AD makes extensive use of such transformations, so `BBCode` is curr There are efforts such as [this PR](https://github.com/JuliaLang/julia/pull/45305) to augment `IRCode` with the capability to manipulate the CFG structure in a convenient manner. Ideally these efforts will succeed, then we can do away with `BBCode`. +### Comparison with Alternative Approaches + +It's worth noting that other automatic differentiation systems have taken different approaches to IR manipulation. For example, [Diffractor.jl](https://github.com/JuliaDiff/Diffractor.jl) uses an "Optics" approach for IR transformations. + +For readers interested in learning more about Julia's IR representation beyond what's covered here, the [Scientific Programming in Julia course materials](https://github.com/JuliaTeachingCTU/Scientific-Programming-in-Julia/blob/2023W/docs/src/lecture_09/ircode.md) provide additional valuable context. + ## Docstrings ```@autodocs; canonical=true diff --git a/docs/src/developer_documentation/reverse_mode_design.md b/docs/src/developer_documentation/reverse_mode_design.md index 8ef4ab190e..04979822ff 100644 --- a/docs/src/developer_documentation/reverse_mode_design.md +++ b/docs/src/developer_documentation/reverse_mode_design.md @@ -72,6 +72,8 @@ Do line by line transformation of the statements and then possibly refresh the C Examples of how line-by-line transformations can be done, are defined in [`Mooncake.make_ad_stmts!`](@ref). The `IRCode` nodes are not explicitly documented in or . Might need completion of official docs, but Mooncake docs in the meantime. +For additional information about `IRCode` and `BBCode` data structures and transformation examples, see [IR Representations and Code Transformations](@ref). + Inlining pass can prevent us from using high-level rules by inlining the function (e.g. unrolling a loop). The contexts in [`interpreter/contexts.jl`](https://github.com/chalk-lab/Mooncake.jl/blob/src/interpreter/contexts.jl) are `MinimalCtx` (necessary for AD to work) and `DefaultCtx` (ensure that we hit all of the rules). Distinction between rules is not well maintained in Mooncake at the moment. From ae79203eb174e3e5f71a746b6f9b8279e569ee47 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 Aug 2025 14:15:37 +0100 Subject: [PATCH 12/12] Add robust Julia version-dependent JET version handling to CI workflow (#733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix changelog version (#693) * remove `const M` (#696) * remove const M * fix formatting * Update Project.toml Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Update README.md (#697) Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * minimal print io output (#699) * minimal print * vers bump * remove prev tests * include show in derived * Update algorithmic_differentiation.md (#703) I noticed a typo. I likely won't look at this again for a week or so, so please feel free to merge for me. Signed-off-by: Will Tebbutt <3628294+willtebbutt@users.noreply.github.com> * Fix 660 (#705) * Fix + tests * Bump patch version * Use _stable_typeof directly * Comment on the use of _stable_typeof * Extend arrayify for complex numbers (#706) * Fix + tests * Bump patch version * Use _stable_typeof directly * Add arrayify tests * Widen permitted types * Bump patch version again * Formatting * Uncomment blas tests * Create contribution guidelines in copilot-instructions.md (#713) Added contribution guidelines for the automatic differentiation package. Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Set tangent_type for AbstractLogger to NoTangent (#710) * Initial plan * Set tangent_type for Logging.ConsoleLogger to NoTangent Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Add ConsoleLogger test to verify AbstractLogger tangent type coverage Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Add Base.CoreLogging.SimpleLogger to type union Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Update test/tangents.jl Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Add better error messages for tangent accessors when types are passed (#707) * Initial plan * Add better error messages for tangent accessors when types are passed Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Hong Ge Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Tidy up (#715) * Tidy up error-generating methods * Remove redundant tests * Remove more redundant test cases * Bump patch version * add rule for complex logdet (#664) * working complex logdet * bump version * proper getrf! tests * improve tests * move tests * generalize tests * formatting and version --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> Signed-off-by: Mateus Araújo Co-authored-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Delete .github/copilot-instructions.md (#722) Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Uncomment b_binv_test_case for PlanarLayer (#727) Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Add DiffEq integration test for Mooncake (#726) * Initial plan * Add DiffEq integration test with original example from issue Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Finalize DiffEq integration test with comprehensive structure Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Remove @info and @test_broken statements to reduce clutter Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * fix formatting (#728) * Fix DiffEq integration test by restricting to reverse mode only (#731) * Initial plan * Fix DiffEq integration test by testing reverse mode only Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * formatting --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> Co-authored-by: Hong Ge * Fix IR docs duplication and improve documentation structure (#718) * Initial plan * Update IR docs title and remove duplication, add comparison section Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Address PR feedback: update title and simplify Optics reference Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Restore IRCode field details that were incorrectly removed Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Initial plan * Add Julia version-dependent JET version selection to CI workflow Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Improve JET version handling to support complex constraints and remove need for explicit version on Julia 1.12+ Co-authored-by: yebai <3279477+yebai@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> * Update CI.yml Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> --------- Signed-off-by: Hong Ge <3279477+yebai@users.noreply.github.com> Signed-off-by: Will Tebbutt <3628294+willtebbutt@users.noreply.github.com> Signed-off-by: Mateus Araújo Co-authored-by: Penelope Yong Co-authored-by: Hong Ge <3279477+yebai@users.noreply.github.com> Co-authored-by: Astitva Aggarwal <84859349+AstitvaAggarwal@users.noreply.github.com> Co-authored-by: Will Tebbutt <3628294+willtebbutt@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: Hong Ge Co-authored-by: Mateus Araújo --- .github/workflows/CI.yml | 43 +++++++++++++ HISTORY.md | 2 +- Project.toml | 3 +- README.md | 2 +- .../ir_representation.md | 8 ++- .../reverse_mode_design.md | 2 + .../algorithmic_differentiation.md | 2 +- src/Mooncake.jl | 1 + src/fwds_rvs_data.jl | 8 +++ src/interpreter/reverse_mode.jl | 9 ++- .../avoiding_non_differentiable_code.jl | 6 +- src/rrules/blas.jl | 15 +++-- src/rrules/lapack.jl | 49 ++++++++++----- src/tangents.jl | 3 + src/tools_for_rules.jl | 10 +-- test/fwds_rvs_data.jl | 5 ++ .../bijectors/bijectors.jl | 2 +- test/integration_testing/diffeq/Project.toml | 6 ++ test/integration_testing/diffeq/diffeq.jl | 63 +++++++++++++++++++ test/interpreter/reverse_mode.jl | 8 +++ test/rrules/blas.jl | 30 ++++++++- 21 files changed, 242 insertions(+), 35 deletions(-) create mode 100644 test/integration_testing/diffeq/Project.toml create mode 100644 test/integration_testing/diffeq/diffeq.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9d4e266024..23eb3309e6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -68,6 +68,27 @@ jobs: arch: ${{ matrix.arch }} include-all-prereleases: false - uses: julia-actions/cache@v2 + - name: Adjust JET version based on Julia version + run: | + julia_version=$(julia --version | grep -oE '[0-9]+\.[0-9]+') + echo "Detected Julia version: $julia_version" + + # Remove any existing JET constraint from [compat] section + sed -i '/^JET = "[0-9]\.[0-9]/d' Project.toml + + # Compare version using version sort + if printf '%s\n' "1.12" "$julia_version" | sort -V | head -n1 | grep -q "1.12"; then + echo "Julia version >= 1.12, using latest JET version (no constraint)" + else + echo "Julia version < 1.12, setting JET = \"0.9\"" + # Add JET = "0.9" constraint in the [compat] section + sed -i '/^\[compat\]/a JET = "0.9"' Project.toml + fi + + # Show the current JET constraint (if any) + echo "Current JET constraint in Project.toml:" + grep -A 20 "^\[compat\]" Project.toml | grep "JET = " || echo "No JET constraint found" + shell: bash - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: @@ -95,6 +116,7 @@ jobs: {test_type: 'integration_testing', label: 'array'}, {test_type: 'integration_testing', label: 'bijectors'}, {test_type: 'integration_testing', label: 'diff_tests'}, + {test_type: 'integration_testing', label: 'diffeq'}, {test_type: 'integration_testing', label: 'dispatch_doctor'}, {test_type: 'integration_testing', label: 'distributions'}, {test_type: 'integration_testing', label: 'dynamicppl'}, @@ -119,6 +141,27 @@ jobs: arch: ${{ matrix.arch }} include-all-prereleases: false - uses: julia-actions/cache@v2 + - name: Adjust JET version based on Julia version + run: | + julia_version=$(julia --version | grep -oE '[0-9]+\.[0-9]+') + echo "Detected Julia version: $julia_version" + + # Remove any existing JET constraint from [compat] section + sed -i '/^JET = "[0-9]\.[0-9]/d' Project.toml + + # Compare version using version sort + if printf '%s\n' "1.12" "$julia_version" | sort -V | head -n1 | grep -q "1.12"; then + echo "Julia version >= 1.12, using latest JET version (no constraint)" + else + echo "Julia version < 1.12, setting JET = \"0.9\"" + # Add JET = "0.9" constraint in the [compat] section + sed -i '/^\[compat\]/a JET = "0.9"' Project.toml + fi + + # Show the current JET constraint (if any) + echo "Current JET constraint in Project.toml:" + grep -A 20 "^\[compat\]" Project.toml | grep "JET = " || echo "No JET constraint found" + shell: bash - uses: julia-actions/julia-buildpkg@v1 - run: | if [ ${{ matrix.test_group.test_type }} == 'ext' ]; then diff --git a/HISTORY.md b/HISTORY.md index 59e82e1a11..3c723f25f9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,4 @@ -# 0.4.143 +# 0.4.147 ## Public Interface - Mooncake offers forward mode AD. diff --git a/Project.toml b/Project.toml index e157394a6a..d500a5d674 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,8 @@ name = "Mooncake" uuid = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" authors = ["Will Tebbutt, Hong Ge, and contributors"] -version = "0.4.149" +version = "0.4.154" + [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" diff --git a/README.md b/README.md index f16806344a..3ae95aa42e 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ using DifferentiationInterface import Mooncake f(x) = sum(cos, x) -backend = AutoMooncake() +backend = AutoMooncake() # Reverse-mode AD. For forward-mode AD, use `AutoMooncakeForward()`. x = ones(1_000) prep = prepare_gradient(f, backend, x) gradient(f, prep, backend, x) diff --git a/docs/src/developer_documentation/ir_representation.md b/docs/src/developer_documentation/ir_representation.md index d9d03c469a..367c20819d 100644 --- a/docs/src/developer_documentation/ir_representation.md +++ b/docs/src/developer_documentation/ir_representation.md @@ -1,4 +1,4 @@ -# IR Representation +# IR Representations and Code Transformations Mooncake.jl works by transforming Julia's SSA-form (static single assignment) Intermediate Representation (IR), so a good understanding of Julia's IR is needed to understand Mooncake. Furthermore, Mooncake holds Julia's IR in a different data structure than the one usually used when producing code for reverse-mode AD. @@ -555,6 +555,12 @@ Reverse-mode AD makes extensive use of such transformations, so `BBCode` is curr There are efforts such as [this PR](https://github.com/JuliaLang/julia/pull/45305) to augment `IRCode` with the capability to manipulate the CFG structure in a convenient manner. Ideally these efforts will succeed, then we can do away with `BBCode`. +### Comparison with Alternative Approaches + +It's worth noting that other automatic differentiation systems have taken different approaches to IR manipulation. For example, [Diffractor.jl](https://github.com/JuliaDiff/Diffractor.jl) uses an "Optics" approach for IR transformations. + +For readers interested in learning more about Julia's IR representation beyond what's covered here, the [Scientific Programming in Julia course materials](https://github.com/JuliaTeachingCTU/Scientific-Programming-in-Julia/blob/2023W/docs/src/lecture_09/ircode.md) provide additional valuable context. + ## Docstrings ```@autodocs; canonical=true diff --git a/docs/src/developer_documentation/reverse_mode_design.md b/docs/src/developer_documentation/reverse_mode_design.md index 8ef4ab190e..04979822ff 100644 --- a/docs/src/developer_documentation/reverse_mode_design.md +++ b/docs/src/developer_documentation/reverse_mode_design.md @@ -72,6 +72,8 @@ Do line by line transformation of the statements and then possibly refresh the C Examples of how line-by-line transformations can be done, are defined in [`Mooncake.make_ad_stmts!`](@ref). The `IRCode` nodes are not explicitly documented in or . Might need completion of official docs, but Mooncake docs in the meantime. +For additional information about `IRCode` and `BBCode` data structures and transformation examples, see [IR Representations and Code Transformations](@ref). + Inlining pass can prevent us from using high-level rules by inlining the function (e.g. unrolling a loop). The contexts in [`interpreter/contexts.jl`](https://github.com/chalk-lab/Mooncake.jl/blob/src/interpreter/contexts.jl) are `MinimalCtx` (necessary for AD to work) and `DefaultCtx` (ensure that we hit all of the rules). Distinction between rules is not well maintained in Mooncake at the moment. diff --git a/docs/src/understanding_mooncake/algorithmic_differentiation.md b/docs/src/understanding_mooncake/algorithmic_differentiation.md index 3fc7050604..f007deceaa 100644 --- a/docs/src/understanding_mooncake/algorithmic_differentiation.md +++ b/docs/src/understanding_mooncake/algorithmic_differentiation.md @@ -5,7 +5,7 @@ Even if you have worked with AD before, we recommend reading in order to acclima # Derivatives -The foundation of automatic differentiation is the directional derivative. +The foundation of automatic differentiation is the derivative. Here we build up to a general definition. _**Scalar-to-Scalar Functions**_ diff --git a/src/Mooncake.jl b/src/Mooncake.jl index afed343bb1..88fa064f52 100644 --- a/src/Mooncake.jl +++ b/src/Mooncake.jl @@ -10,6 +10,7 @@ import ChainRulesCore as CRC using Base: IEEEFloat, + ReshapedArray, unsafe_convert, unsafe_pointer_to_objref, pointer_from_objref, diff --git a/src/fwds_rvs_data.jl b/src/fwds_rvs_data.jl index 0ad4981e3e..2cf7bceee6 100644 --- a/src/fwds_rvs_data.jl +++ b/src/fwds_rvs_data.jl @@ -258,6 +258,10 @@ function fdata(t::T) where {T} return F(fdata(t.fields)) end +function fdata(::Type{T}) where {T} + error("$T is a type. Perhaps you meant fdata_type($T) or fdata(instance_of_tangent)?") +end + function fdata(t::T) where {T<:PossiblyUninitTangent} F = fdata_type(T) return is_init(t) ? F(fdata(val(t))) : F() @@ -530,6 +534,10 @@ function rdata(t::T) where {T} return R(rdata(t.fields)) end +function rdata(::Type{T}) where {T} + error("$T is a type. Perhaps you meant rdata_type($T) or rdata(instance_of_tangent)?") +end + function rdata(t::T) where {T<:PossiblyUninitTangent} R = rdata_type(T) return is_init(t) ? R(rdata(val(t))) : R() diff --git a/src/interpreter/reverse_mode.jl b/src/interpreter/reverse_mode.jl index af989bc248..bb7fb941f6 100644 --- a/src/interpreter/reverse_mode.jl +++ b/src/interpreter/reverse_mode.jl @@ -1746,7 +1746,14 @@ DynamicDerivedRule(debug_mode::Bool) = DynamicDerivedRule(Dict{Any,Any}(), debug _copy(x::P) where {P<:DynamicDerivedRule} = P(Dict{Any,Any}(), x.debug_mode) function (dynamic_rule::DynamicDerivedRule)(args::Vararg{Any,N}) where {N} - sig = Tuple{map(_typeof ∘ primal, args)...} + + # `Base._stable_typeof` is used here, rather than `typeof` or `Mooncake._typeof`. Its + # precise behaviour (equivalent to `typeof` for everything except `Type`s, for which it + # returns `Type{P}` rather than `typeof(P)`) is needed to ensure that this signature + # matches the types that `rule` sees when `rule(args...)` is called below. If you get + # this wrong, an assertion is violated, causing a hard-to-debug error (see issue 660). + sig = Tuple{map(Base._stable_typeof ∘ primal, args)...} + rule = get(dynamic_rule.cache, sig, nothing) if rule === nothing interp = get_interpreter(ReverseMode) diff --git a/src/rrules/avoiding_non_differentiable_code.jl b/src/rrules/avoiding_non_differentiable_code.jl index 11a31e0820..46ee48867e 100644 --- a/src/rrules/avoiding_non_differentiable_code.jl +++ b/src/rrules/avoiding_non_differentiable_code.jl @@ -138,9 +138,6 @@ function generate_hand_written_rrule!!_test_cases( ("hello" => "hi",), 1, ), - (false, :none, nothing, print, "Testing print"), - (false, :none, nothing, println, "Testing println"), - (false, :none, nothing, show, "Testing show"), # non-kwargs sprint rule test (false, :stability, nothing, sprint, show, "Testing sprint"), @@ -202,6 +199,9 @@ function generate_derived_rrule!!_test_cases( test_cases = vcat( Any[ # Tests for Base.CoreLogging, @show macros and string related functions. + (false, :none, nothing, (x) -> print(x), "Testing print"), + (false, :none, nothing, (x) -> println(x), "Testing println"), + (false, :none, nothing, (x) -> show(x), "Testing show"), (false, :none, nothing, testloggingmacro1, rand(1:100)), (false, :none, nothing, testloggingmacro2, rand(1:100)), (false, :none, nothing, testloggingmacro3, rand(1:100)), diff --git a/src/rrules/blas.jl b/src/rrules/blas.jl index aa63cad7c5..f7fd44a9e5 100644 --- a/src/rrules/blas.jl +++ b/src/rrules/blas.jl @@ -35,13 +35,13 @@ arrayify(x::Array{P}, dx::Array{P}) where {P<:BlasRealFloat} = (x, dx) function arrayify(x::Array{P}, dx::Array{<:Tangent}) where {P<:BlasComplexFloat} return x, reinterpret(P, dx) end -function arrayify(x::A, dx::TangentOrFData) where {A<:SubArray{<:BlasRealFloat}} +function arrayify(x::SubArray{P,B,C,D,E}, dx::TangentOrFData) where {P<:BlasFloat,B,C,D,E} _, _dx = arrayify(x.parent, _fields(dx).parent) - return x, A(_dx, x.indices, x.offset1, x.stride1) + return x, SubArray{P,B,typeof(_dx),D,E}(_dx, x.indices, x.offset1, x.stride1) end -function arrayify(x::A, dx::TangentOrFData) where {A<:Base.ReshapedArray{<:BlasRealFloat}} +function arrayify(x::ReshapedArray{P,B,C,D}, dx::TangentOrFData) where {P<:BlasFloat,B,C,D} _, _dx = arrayify(x.parent, _fields(dx).parent) - return x, A(_dx, x.dims, x.mi) + return x, ReshapedArray{P,B,typeof(_dx),D}(_dx, x.dims, x.mi) end function arrayify(x::Base.ReinterpretArray{T}, dx::TangentOrFData) where {T<:BlasFloat} _, _dx = arrayify(x.parent, _fields(dx).parent) @@ -1109,6 +1109,13 @@ function invertible_blas_matrices(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int end end +function positive_definite_blas_matrices(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int) + return map(blas_matrices(rng, P, p, p)) do A + A .= A'A + I + return A + end +end + function blas_vectors(rng::AbstractRNG, P::Type{<:BlasFloat}, p::Int) xs = Any[ randn(rng, P, p), diff --git a/src/rrules/lapack.jl b/src/rrules/lapack.jl index ede01c2217..2d5229e21c 100644 --- a/src/rrules/lapack.jl +++ b/src/rrules/lapack.jl @@ -1,14 +1,14 @@ # See https://sethaxen.com/blog/2021/02/differentiating-the-lu-decomposition/ for details. -@is_primitive(MinimalCtx, Tuple{typeof(LAPACK.getrf!),AbstractMatrix{<:BlasRealFloat}}) +@is_primitive(MinimalCtx, Tuple{typeof(LAPACK.getrf!),AbstractMatrix{<:BlasFloat}}) function frule!!( ::Dual{typeof(LAPACK.getrf!)}, A_dA::Dual{<:AbstractMatrix{P}} -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} _, ipiv, info = LAPACK.getrf!(primal(A_dA)) return _getrf_fwd(A_dA, ipiv, info) end function rrule!!( ::CoDual{typeof(LAPACK.getrf!)}, _A::CoDual{<:AbstractMatrix{P}} -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} A, dA = arrayify(_A) A_copy = copy(A) @@ -28,16 +28,14 @@ end @is_primitive( MinimalCtx, - Tuple{ - typeof(Core.kwcall),NamedTuple,typeof(LAPACK.getrf!),AbstractMatrix{<:BlasRealFloat} - }, + Tuple{typeof(Core.kwcall),NamedTuple,typeof(LAPACK.getrf!),AbstractMatrix{<:BlasFloat}}, ) function frule!!( ::Dual{typeof(Core.kwcall)}, _kwargs::Dual{<:NamedTuple}, ::Dual{typeof(getrf!)}, A_dA::Dual{<:AbstractMatrix{P}}, -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} check = primal(_kwargs).check _, ipiv, info = LAPACK.getrf!(primal(A_dA); check) return _getrf_fwd(A_dA, ipiv, info) @@ -47,7 +45,7 @@ function rrule!!( _kwargs::CoDual{<:NamedTuple}, ::CoDual{typeof(getrf!)}, _A::CoDual{<:AbstractMatrix{P}}, -) where {P<:BlasRealFloat} +) where {P<:BlasFloat} check = _kwargs.x.check A, dA = arrayify(_A) A_copy = copy(A) @@ -505,6 +503,7 @@ end function generate_hand_written_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) rng = rng_ctor(123) Ps = [Float64, Float32] + complexPs = [Float64, Float32, ComplexF64, ComplexF32] bools = [false, true] test_cases = vcat( @@ -516,7 +515,7 @@ function generate_hand_written_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) (false, :stability, nothing, getrf!, A) end end..., - map_prod(bools, Ps) do (check, P) + map_prod(bools, complexPs) do (check, P) As = blas_matrices(rng, P, 5, 5) ipiv = Vector{Int}(undef, 5) return map(As) do A @@ -583,13 +582,33 @@ end function generate_derived_rrule!!_test_cases(rng_ctor, ::Val{:lapack}) rng = rng_ctor(123) + complexPs = [Float64, Float32, ComplexF64, ComplexF32] getrf_wrapper!(x, check) = getrf!(x; check) - test_cases = vcat(map_prod([false, true], [Float64, Float32]) do (check, P) - As = blas_matrices(rng, P, 5, 5) - return map(As) do A - (false, :none, nothing, getrf_wrapper!, A, check) - end - end...) + test_cases = vcat( + # getrf + map_prod([false, true], complexPs) do (check, P) + As = blas_matrices(rng, P, 5, 5) + return map(As) do A + (false, :none, nothing, getrf_wrapper!, A, check) + end + end..., + + # real logdet + map([Float64, Float32]) do P + As = positive_definite_blas_matrices(rng, P, 3) + return map(As) do A + (false, :none, nothing, logdet, A) + end + end..., + + # complex logdet + map(complexPs) do P + As = blas_matrices(rng, P, 3, 3) + return map(As) do A + (false, :none, nothing, real ∘ logdet ∘ complex, A) + end + end..., + ) memory = Any[] return test_cases, memory end diff --git a/src/tangents.jl b/src/tangents.jl index 73a40371b6..5f6c26be51 100644 --- a/src/tangents.jl +++ b/src/tangents.jl @@ -370,6 +370,8 @@ tangent_type(::Type{<:Base.TTY}) = NoTangent tangent_type(::Type{<:IOStream}) = NoTangent +tangent_type(::Type{<:Base.CoreLogging.AbstractLogger}) = NoTangent + function split_union_tuple_type(tangent_types) # Create first split. @@ -1353,6 +1355,7 @@ tangents, but they're unable to check that `increment!!` is correct in an absolu (Union, NoTangent), (UnionAll, NoTangent), (typeof(<:), NoTangent), + (Base.CoreLogging.SimpleLogger, NoTangent), (IOStream(""), NoTangent), ] # Construct test cases containing circular references. These typically require multiple diff --git a/src/tools_for_rules.jl b/src/tools_for_rules.jl index f747121154..820b711365 100644 --- a/src/tools_for_rules.jl +++ b/src/tools_for_rules.jl @@ -282,9 +282,8 @@ function _zero_derivative_impl(ctx, sig, mode) # which does not escape the mode argument. This will work even if the names `Mooncake` # or `Mooncake.Mode` are not available in the scope which calls this macro. is_primitive_ex = quote - const M = $mode function Mooncake.is_primitive( - ::Type{$(esc(ctx))}, ::Type{<:M}, ::Type{<:$(esc(sig))} + ::Type{$(esc(ctx))}, ::Type{<:$mode}, ::Type{<:$(esc(sig))} ) return true end @@ -605,7 +604,9 @@ function _from_chainrules_impl(ctx, sig::Expr, has_kwargs::Bool, mode) kw_sig = where_params === nothing ? kw_sig : Expr(:where, kw_sig, where_params...) # Type M will be available later on, and will be the mode type. kw_is_primitive = quote - function Mooncake.is_primitive(::Type{$(esc(ctx))}, ::Type{<:M}, ::Type{<:$kw_sig}) + function Mooncake.is_primitive( + ::Type{$(esc(ctx))}, ::Type{<:$mode}, ::Type{<:$kw_sig} + ) return true end end @@ -634,9 +635,8 @@ function _from_chainrules_impl(ctx, sig::Expr, has_kwargs::Bool, mode) end return quote - const M = $mode function Mooncake.is_primitive( - ::Type{$(esc(ctx))}, ::Type{<:M}, ::Type{<:($(esc(sig)))} + ::Type{$(esc(ctx))}, ::Type{<:$mode}, ::Type{<:($(esc(sig)))} ) return true end diff --git a/test/fwds_rvs_data.jl b/test/fwds_rvs_data.jl index ebe485ac1c..63f775caf9 100644 --- a/test/fwds_rvs_data.jl +++ b/test/fwds_rvs_data.jl @@ -157,4 +157,9 @@ end @test verify_rdata_value(Ptr{Float64}(), NoRData()) === nothing end end + + @testset "Helpful error messages for misuse of fdata and rdata" begin + @test_throws "Float64 is a type. Perhaps you meant" fdata(Float64) + @test_throws "Float64 is a type. Perhaps you meant" rdata(Float64) + end end diff --git a/test/integration_testing/bijectors/bijectors.jl b/test/integration_testing/bijectors/bijectors.jl index 998c1f34d0..289f97c6d5 100644 --- a/test/integration_testing/bijectors/bijectors.jl +++ b/test/integration_testing/bijectors/bijectors.jl @@ -50,7 +50,7 @@ end 1 0 0 0 0 1 ]), (3, 3)), - # b_binv_test_case(Bijectors.PlanarLayer(3), (3, 3)), + b_binv_test_case(Bijectors.PlanarLayer(3), (3, 3)), b_binv_test_case(Bijectors.RadialLayer(3), 3), b_binv_test_case(Bijectors.Reshape((2, 3), (3, 2)), (2, 3)), b_binv_test_case(Bijectors.Scale(0.2), 3), diff --git a/test/integration_testing/diffeq/Project.toml b/test/integration_testing/diffeq/Project.toml new file mode 100644 index 0000000000..98e015ff48 --- /dev/null +++ b/test/integration_testing/diffeq/Project.toml @@ -0,0 +1,6 @@ +[deps] +Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" +OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" +StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" \ No newline at end of file diff --git a/test/integration_testing/diffeq/diffeq.jl b/test/integration_testing/diffeq/diffeq.jl new file mode 100644 index 0000000000..b998d5fe41 --- /dev/null +++ b/test/integration_testing/diffeq/diffeq.jl @@ -0,0 +1,63 @@ +using Pkg +Pkg.activate(@__DIR__) +Pkg.develop(; path=joinpath(@__DIR__, "..", "..", "..")) + +using OrdinaryDiffEq, SciMLSensitivity, Mooncake, StableRNGs, Test +using Mooncake.TestUtils: test_rule +using Mooncake: ReverseMode + +# Helper function for Mooncake gradient computation +function mooncake_gradient(f, x) + Mooncake.value_and_gradient!!(Mooncake.build_rrule(f, x), f, x)[2][2] +end + +# Define the ODE function from the original issue +odef(du, u, p, t) = du .= u .* p + +# Define the sensitivity loss function from the original issue +struct senseloss0{T} + sense::T +end + +function (f::senseloss0)(u0p) + prob = ODEProblem{true}(odef, u0p[1:1], (0.0, 1.0), u0p[2:2]) + sum(solve(prob, Tsit5(); abstol=1e-12, reltol=1e-12, saveat=0.1)) +end + +@testset "diffeq" begin + rng = StableRNG(123456) + + # Test parameters from the original issue + u0p = [2.0, 3.0] + + @testset "senseloss0 with InterpolatingAdjoint" begin + sense_func = senseloss0(InterpolatingAdjoint()) + + # First test that the function works + @testset "Function evaluation" begin + result = sense_func(u0p) + @test result isa Real + @test isfinite(result) + end + + # Test Mooncake gradient computation + @testset "mooncake_gradient computation" begin + dup_mc = mooncake_gradient(sense_func, u0p) + @test dup_mc isa Vector + @test length(dup_mc) == 2 + @test all(isfinite, dup_mc) + end + + # Test with Mooncake's test_rule + @testset "test_rule evaluation" begin + test_rule( + rng, + sense_func, + u0p; + is_primitive=false, + unsafe_perturb=true, + mode=ReverseMode, + ) + end + end +end diff --git a/test/interpreter/reverse_mode.jl b/test/interpreter/reverse_mode.jl index 8b930f2a4b..b1986ffdab 100644 --- a/test/interpreter/reverse_mode.jl +++ b/test/interpreter/reverse_mode.jl @@ -14,6 +14,9 @@ f(a, x) = dot(a.data, x) unstable_tester(x::Ref{Any}) = sin(x[]) +# used for regression test for issue 660 +struct MakeAUnionAll{T} end + end @testset "s2s_reverse_mode_ad" begin @@ -353,6 +356,11 @@ end f(x) = sin(cos(x)) rule = Mooncake.build_rrule(f, 0.0) @benchmark Mooncake.value_and_gradient!!($rule, $f, $(Ref(0.0))[]) + + # 660 -- ensure that the correct signature is used to construct DynamicDerivedRules + rule = Mooncake.DynamicDerivedRule(false) + args = (zero_fcodual(identity), zero_fcodual((v=S2SGlobals.MakeAUnionAll,))) + @test rule(args...) isa Tuple{CoDual,Any} end @testset "literal Strings do not appear in shared data" begin f() = "hello" diff --git a/test/rrules/blas.jl b/test/rrules/blas.jl index 09629a18e4..0a3845c512 100644 --- a/test/rrules/blas.jl +++ b/test/rrules/blas.jl @@ -1,4 +1,32 @@ @testset "blas" begin - @test_throws "Encountered unexpected array type" Mooncake.arrayify(5, 4) + + # Problems with arrayify tend to get picked up by tests for rules which use arrayify. + # However, such problems can be quite tricky to diagnose from these failures alone, so + # it is worth unit testing some properties (eg. aliasing). + @testset "arrayify" begin + + # Verify that an unexpected type throws a sensible error. + @test_throws "Encountered unexpected array type" Mooncake.arrayify(5, 4) + + # Verify all test cases can be array-ified. + @testset "$P" for P in [Float32, Float64, ComplexF32, ComplexF64] + xs = Mooncake.blas_matrices(StableRNG(123), P, 2, 3) + @testset "$(typeof(x)), $f" for x in xs, f in [identity, fdata] + t = f(Mooncake.randn_tangent(StableRNG(123), x)) + _x, _t = Mooncake.arrayify(Mooncake.CoDual(x, t)) + + # The primal should be the same thing. + @test _x === x + + # The data underlying the tangent / fdata returned from arrayify must alias + # the original. To check that this happens, we check that if we run arrayify a + # second time on the same input, and mutate the tangent, the values in `_t` + # are modified in exactly the same way. + _, _t2 = Mooncake.arrayify(Mooncake.CoDual(x, t)) + _t2 .= zero(P) + @test _t == _t2 + end + end + end TestUtils.run_rule_test_cases(StableRNG, Val(:blas)) end