From 2e596833741988f8645447ac02348866a23fb0d3 Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 09:58:16 +0200 Subject: [PATCH 1/8] document QuantilePoint --- docs/make.jl | 11 +++++---- docs/src/api.md | 25 +++++++++++++++++++ docs/src/index.md | 6 ----- src/DistributionFits.jl | 3 +++ src/fitstats.jl | 53 +++++++++++++++++++++++++++++++---------- 5 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 docs/src/api.md diff --git a/docs/make.jl b/docs/make.jl index e5e424e..2d8ed22 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -21,10 +21,13 @@ makedocs(; pages=[ "Home" => "index.md", "Dependencies" => "set_optimize.md", - "LogNormal" => "lognormal.md", - "LogitNormal" => "logitnormal.md", - "Weibull" => "weibull.md", - "Gamma" => "gamma.md", + "Distributions" => [ + "LogNormal" => "lognormal.md", + "LogitNormal" => "logitnormal.md", + "Weibull" => "weibull.md", + "Gamma" => "gamma.md", + ], + "API" => "api.md", #"Details" => "z_autodocs.md", ], ) diff --git a/docs/src/api.md b/docs/src/api.md new file mode 100644 index 0000000..55f9c19 --- /dev/null +++ b/docs/src/api.md @@ -0,0 +1,25 @@ +# API +## AbstractMoments + +The concept of first and higher order moments is +captured by its own type. This allows dispatching +the fit method. + +```@docs +AbstractMoments +moments +``` + +## QuantilePoint + +The concept of a pair (p,q), i.e. a probability in [0,1] and associated quantile +is captured by its own type. This allows dispatching +the fit method. + +```@docs +QuantilePoint +``` + + + + diff --git a/docs/src/index.md b/docs/src/index.md index de2c3b3..ddf1df0 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -77,12 +77,6 @@ fit(::Type{D}, ::AbstractMoments) where {D<:Distribution} moments(d::Distribution, ::Val{N} = Val(2)) where N ``` -The syntax `Moments(mean,var)` produces an object of type `Moments <: AbstractMoments`. - -```@docs - AbstractMoments{N} -``` - ## Fit to several quantile points ```@docs diff --git a/src/DistributionFits.jl b/src/DistributionFits.jl index 34e288b..9395e65 100644 --- a/src/DistributionFits.jl +++ b/src/DistributionFits.jl @@ -5,6 +5,7 @@ using Reexport using FillArrays, StaticArrays using StatsFuns: logit, logistic, normcdf +#using Infiltrator if !isdefined(Base, :get_extension) using Requires @@ -22,6 +23,8 @@ export fit_mean_quantile, fit_mode_quantile, fit_median_quantile, @qp, @qp_ll, @qp_l, @qp_m, @qp_u, @qp_uu, @qs_cf90, @qs_cf95, + qp, qp_ll, qp_l, qp_m, qp_u, qp_uu, + qs_cf90, qs_cf95, fit_mean_relerror # document but do not export - need to qualify by 'DistributionFits.' diff --git a/src/fitstats.jl b/src/fitstats.jl index d976223..07b984a 100644 --- a/src/fitstats.jl +++ b/src/fitstats.jl @@ -58,7 +58,7 @@ Base.convert(::Type{AbstractArray}, m::Moments) = m.all Get the first N moments of a distribution. -See also type [`AbstractMoments`](@ref). +Procudes an object of type [`AbstractMoments`](@ref). ## Examples ```julia @@ -116,7 +116,28 @@ fit(::Type{D}, m::AbstractMoments) where {D<:Distribution} = error("fitting to moments not implemented for distribution of type $D") +""" + QuantilePoint + +A representation of a pair (p,q), i.e. (percentile,quantile). +# Notes +Several macros help to construct QuantilePoints +- `@qp(q,p)` quantile at specified p: `QuantilePoint(q,p)` + +For Float64-based percentiles there are shortcuts +- `@qp_ll(q0_025)` quantile at very low p: `QuantilePoint(q0_025,0.025)` +- `@qp_l(q0_05)` quantile at low p: `QuantilePoint(q0_05,0.05)` +- `@qp_m(median)` quantile at median: `QuantilePoint(median,0.5)` +- `@qp_u(q0_95)` quantile at high p: `QuantilePoint(q0_95,0.95)` +- `@qp_uu(q0_975)` quantile at very high p: `QuantilePoint(q0_975,0.975)` + +For constructing QuantilePoints with type of percentiles other than Float64, +use the corresponding functions, that create a percentiles of the type +of qiven quantile. +E.g. for a Float32-based QuantilePoint at ver low percentile +- `qp_ll(0.2f0)` constructs a `QuantilePoint(0.2f0,0.025f0)` +""" struct QuantilePoint{TQ,TP} q::TQ p::TP @@ -149,6 +170,20 @@ macro qs_cf90(q0_05,q0_95) macro qs_cf95(q0_025,q0_975) :(Set([QuantilePoint($(esc(q0_025)),0.025),QuantilePoint($(esc(q0_975)),0.975)])) end +# The non-macro versions return percentile whose type matches that of the argument +qp_ll(q0_025::T) where T = QuantilePoint(q0_025, T(0.025)) +qplm(q0_05::T) where T = QuantilePoint(q0_05, T(0.05)) +qp_m(median::T) where T = QuantilePoint(median, T(0.5)) +qp_u(q0_95::T) where T = QuantilePoint(q0_95, T(0.95)) +qp_uu(q0_975::T) where T = QuantilePoint(q0_975, T(0.975)) + +function qs_cf90(q0_05::T,q0_95::T) where T + Set([QuantilePoint(q0_05,T(0.05)),QuantilePoint(q0_95,T(0.95))]) +end +function qs_cf95(q0_025::T,q0_975::T) where T + Set([QuantilePoint(q0_025,T(0.025)),QuantilePoint(q0_975,T(0.975))]) +end + """ fit(D, lower, upper) @@ -157,18 +192,9 @@ Fit a statistical distribution to a set of quantiles # Arguments - `D`: The type of the distribution to fit -- `lower`: lower QuantilePoint (p,q) +- `lower`: lower [`QuantilePoint`](@ref) (p,q) - `upper`: upper QuantilePoint (p,q) -# Notes -Several macros help to construct QuantilePoints -- `@qp(q,p)` quantile at specified p: `QuantilePoint(q,p)` -- `@qp_ll(q0_025)` quantile at very low p: `QuantilePoint(q0_025,0.025)` -- `@qp_l(q0_05)` quantile at low p: `QuantilePoint(q0_05,0.05)` -- `@qp_m(median)` quantile at median: `QuantilePoint(median,0.5)` -- `@qp_u(q0_95)` quantile at high p: `QuantilePoint(q0_95,0.95)` -- `@qp_uu(q0_975)` quantile at very high p: `QuantilePoint(q0_975,0.975)` - # Examples ```jldoctest; output = false, setup = :(using Statistics,Distributions) d = fit(LogNormal, @qp_m(3), @qp_uu(5)); @@ -217,7 +243,10 @@ function fit(::Type{D}, val, qp::QuantilePoint, ::Val{stats} = Val(:mean)) where stats == :median && return(fit_median_quantile(D, val, qp)) error("unknown stats: $stats") end, -function fit_mean_quantile(::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution +# function fit_mean_quantile(d::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution +# error("fit_mean_quantile not yet implemented for Distribution of type: $D") +# end +function fit_mean_quantile(d::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution error("fit_mean_quantile not yet implemented for Distribution of type: $D") end function fit_mode_quantile(::Type{D}, mode::Real, qp::QuantilePoint) where D<:Distribution From c764495eba24862ee21515be57b2abd0293d7f0d Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 10:46:48 +0200 Subject: [PATCH 2/8] remove autodocs that generates errors of duplicated entries during deplay_docs --- docs/src/z_autodocs.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/z_autodocs.md b/docs/src/z_autodocs.md index 29d7044..7f2bf60 100644 --- a/docs/src/z_autodocs.md +++ b/docs/src/z_autodocs.md @@ -9,6 +9,8 @@ Documentation for [DistributionFits](https://github.com/bgctw/DistributionFits.j ```@index ``` + From 5a3f718b3d966235afca0eb89c4df8160ae4e2df Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 10:54:39 +0200 Subject: [PATCH 3/8] document cf90 and cf95 confidence intervals --- src/fitstats.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fitstats.jl b/src/fitstats.jl index 07b984a..295886f 100644 --- a/src/fitstats.jl +++ b/src/fitstats.jl @@ -137,6 +137,10 @@ use the corresponding functions, that create a percentiles of the type of qiven quantile. E.g. for a Float32-based QuantilePoint at ver low percentile - `qp_ll(0.2f0)` constructs a `QuantilePoint(0.2f0,0.025f0)` + +There are macros/functions for some commonly used sets of QuantilePoints: 90% and 95% confidence intervals: +- `@qs_cf90(q0_05,q0_95)` +- `@qs_cf95(q0_025,q0_975)` -> `Set([QuantilePoint(q0_025,0.025),QuantilePoint(q0_975,0.975)]))` """ struct QuantilePoint{TQ,TP} q::TQ From cfd440b470d65d61a37a2a458bdb175d54526817 Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 11:03:15 +0200 Subject: [PATCH 4/8] implement tests for qp functions --- src/fitstats.jl | 2 +- test/fitstats.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/fitstats.jl b/src/fitstats.jl index 295886f..ac828cc 100644 --- a/src/fitstats.jl +++ b/src/fitstats.jl @@ -176,7 +176,7 @@ macro qs_cf95(q0_025,q0_975) # The non-macro versions return percentile whose type matches that of the argument qp_ll(q0_025::T) where T = QuantilePoint(q0_025, T(0.025)) -qplm(q0_05::T) where T = QuantilePoint(q0_05, T(0.05)) +qp_l(q0_05::T) where T = QuantilePoint(q0_05, T(0.05)) qp_m(median::T) where T = QuantilePoint(median, T(0.5)) qp_u(q0_95::T) where T = QuantilePoint(q0_95, T(0.95)) qp_uu(q0_975::T) where T = QuantilePoint(q0_975, T(0.975)) diff --git a/test/fitstats.jl b/test/fitstats.jl index 7413df4..d358145 100644 --- a/test/fitstats.jl +++ b/test/fitstats.jl @@ -82,8 +82,20 @@ end; @test @qs_cf90(0.2,0.7) == Set([@qp_l(0.2),@qp_u(0.7)]) @test @qs_cf95(0.2,0.7) == Set([@qp_ll(0.2),@qp_uu(0.7)]) end; + @testset "functions for Float32" begin + @test @qp(0.4f0,0.7f0) == QuantilePoint(0.4f0,0.7f0) + @test qp_ll(0.7f0) == QuantilePoint(0.7f0,0.025f0) + @test qp_l(0.7f0) == QuantilePoint(0.7f0,0.05f0) + @test qp_m(0.7f0) == QuantilePoint(0.7f0,0.5f0) + @test qp_u(0.7f0) == QuantilePoint(0.7f0,0.95f0) + @test qp_uu(0.7f0) == QuantilePoint(0.7f0,0.975f0) + @test qs_cf90(0.2f0,0.7f0) == Set([qp_l(0.2f0),qp_u(0.7f0)]) + @test qs_cf95(0.2f0,0.7f0) == Set([qp_ll(0.2f0),qp_uu(0.7f0)]) + end; end; + + @testset "global fit functions" begin @test_throws ErrorException fit(Distribution, Moments()) qpl = @qp_m(3) From 0bec154b088b1065d070ec93adea9f7328492732 Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 11:09:52 +0200 Subject: [PATCH 5/8] markdown comment autodocs --- docs/src/z_autodocs.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/z_autodocs.md b/docs/src/z_autodocs.md index 7f2bf60..d043e46 100644 --- a/docs/src/z_autodocs.md +++ b/docs/src/z_autodocs.md @@ -9,8 +9,8 @@ Documentation for [DistributionFits](https://github.com/bgctw/DistributionFits.j ```@index ``` - +[comment]: # (```@autodocs) + +[comment]: # (Modules = [DistributionFits]) + +[comment]: # (```) From 23ee104694fa5fc1dee46b9e91c93fe87e93fa1f Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 11:45:29 +0200 Subject: [PATCH 6/8] include docrefs explicitly in z_autodocs --- docs/src/z_autodocs.md | 17 +++++++++++++++++ src/fitstats.jl | 18 ++++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/src/z_autodocs.md b/docs/src/z_autodocs.md index d043e46..6b58898 100644 --- a/docs/src/z_autodocs.md +++ b/docs/src/z_autodocs.md @@ -14,3 +14,20 @@ Documentation for [DistributionFits](https://github.com/bgctw/DistributionFits.j [comment]: # (Modules = [DistributionFits]) [comment]: # (```) + + +taken care of in index.md +called only internally from fit, documented in docstring via "," +```@docs +fit_median_quantile +fit_mean_quantile +fit_mode_quantile +``` + +internals +```@docs +meanFunOfProb +ofLogitNormalModeUpper +fit(Type{LogNormal}, Any, AbstractΣstar) +``` + diff --git a/src/fitstats.jl b/src/fitstats.jl index ac828cc..40abcf9 100644 --- a/src/fitstats.jl +++ b/src/fitstats.jl @@ -190,7 +190,7 @@ end """ - fit(D, lower, upper) + fit(D, lower::QuantilePoint, upper::QuantilePoint) Fit a statistical distribution to a set of quantiles @@ -209,12 +209,10 @@ true """ function fit(::Type{D}, lower::QuantilePoint, upper::QuantilePoint) where D<:Distribution error("fitting to two quantile points not implemented for distribution of type $D") -end, -function fit_median_quantile(D::Type{DT}, median, qp::QuantilePoint) where {DT <: Distribution} - return(fit(D, @qp_m(median), qp)) end + """ fit(D, val, qp, ::Val{stats} = Val(:mean)) @@ -247,15 +245,19 @@ function fit(::Type{D}, val, qp::QuantilePoint, ::Val{stats} = Val(:mean)) where stats == :median && return(fit_median_quantile(D, val, qp)) error("unknown stats: $stats") end, -# function fit_mean_quantile(d::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution -# error("fit_mean_quantile not yet implemented for Distribution of type: $D") -# end +function fit_median_quantile(D::Type{DT}, median, qp::QuantilePoint) where {DT <: Distribution} + return(fit(D, @qp_m(median), qp)) +end, function fit_mean_quantile(d::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution error("fit_mean_quantile not yet implemented for Distribution of type: $D") -end +end, function fit_mode_quantile(::Type{D}, mode::Real, qp::QuantilePoint) where D<:Distribution error("fit_mode_quantile not yet implemented for Distribution of type: $D") end +# function fit_mean_quantile(d::Type{D}, mean::Real, qp::QuantilePoint) where D<:Distribution +# error("fit_mean_quantile not yet implemented for Distribution of type: $D") +# end + From c87f9a52b92c0baaa24536f541e41108cde08abc Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 12:14:34 +0200 Subject: [PATCH 7/8] include StatsAPI in dependencies --- Project.toml | 16 ++++++++-------- src/DistributionFits.jl | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Project.toml b/Project.toml index 5dd9e20..afc1776 100644 --- a/Project.toml +++ b/Project.toml @@ -10,9 +10,15 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69" Requires = "ae029012-a4dd-5104-9daa-d747884805df" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +StatsAPI = "82ae8749-77ed-4fe6-ae5f-f523153014b0" StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +[weakdeps] +Optim = "429524aa-4258-5aef-a3af-852621145aeb" + +[extensions] +DistributionFitsOptimExt = "Optim" + [compat] Distributions = "0.25" FillArrays = "0.12, 0.13, 1" @@ -20,15 +26,9 @@ Optim = "1.7" Reexport = "1.2" Requires = "1.2" StaticArrays = "1.2" -StatsBase = "0.33, 0.34" StatsFuns = "0.9, 1" +StatsAPI = "1" julia = "1.6" -[extensions] -DistributionFitsOptimExt = "Optim" - [extras] Optim = "429524aa-4258-5aef-a3af-852621145aeb" - -[weakdeps] -Optim = "429524aa-4258-5aef-a3af-852621145aeb" diff --git a/src/DistributionFits.jl b/src/DistributionFits.jl index 9395e65..8d4b67f 100644 --- a/src/DistributionFits.jl +++ b/src/DistributionFits.jl @@ -13,7 +13,8 @@ end # for extension import Distributions: mean, var, mode -import StatsBase: fit +#import StatsBase: fit +import StatsAPI: fit # Moments also extends getindex, mean, kurtorsis .... export From 80052fd8b1e1fb791a6828cce045e13818754035 Mon Sep 17 00:00:00 2001 From: Thomas Wutzler Date: Fri, 22 Sep 2023 12:15:20 +0200 Subject: [PATCH 8/8] update docs of LogNormal to widen type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Σstar -> AbstractΣstar --- docs/src/lognormal.md | 2 +- docs/src/z_autodocs.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/lognormal.md b/docs/src/lognormal.md index 1ee705d..048bd65 100644 --- a/docs/src/lognormal.md +++ b/docs/src/lognormal.md @@ -37,7 +37,7 @@ true ``` ```@docs -fit(::Type{LogNormal}, ::Any, ::Σstar) +fit(::Type{LogNormal}, ::Any, ::AbstractΣstar) ``` ```@docs diff --git a/docs/src/z_autodocs.md b/docs/src/z_autodocs.md index 6b58898..f619fc8 100644 --- a/docs/src/z_autodocs.md +++ b/docs/src/z_autodocs.md @@ -28,6 +28,6 @@ internals ```@docs meanFunOfProb ofLogitNormalModeUpper -fit(Type{LogNormal}, Any, AbstractΣstar) ``` +[comment]: # (fit(Type{LogNormal}, Any, AbstractΣstar))