From 25a290e320de630eaf8ef4e56bb435c1ecfe1032 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 1 Jan 2022 16:26:27 +0800 Subject: [PATCH 01/22] Upgrade bleve from v2.0.6 to v2.3.0 (#18132) --- go.mod | 3 +- go.sum | 52 +++---- modules/indexer/code/bleve.go | 2 +- .../RoaringBitmap/roaring/arraycontainer.go | 2 +- .../RoaringBitmap/roaring/fastaggregation.go | 22 +-- .../github.com/RoaringBitmap/roaring/go.mod | 2 - .../github.com/RoaringBitmap/roaring/go.sum | 4 - .../RoaringBitmap/roaring/roaring.go | 24 ++++ .../roaring/serialization_littleendian.go | 7 +- .../blevesearch/bleve/v2/.gitignore | 1 + .../blevesearch/bleve/v2/analysis/type.go | 1 + .../blevesearch/bleve/v2/config_app.go | 1 + .../blevesearch/bleve/v2/config_disk.go | 1 + .../blevesearch/bleve/v2/document/field_ip.go | 132 ++++++++++++++++++ vendor/github.com/blevesearch/bleve/v2/go.mod | 22 +-- vendor/github.com/blevesearch/bleve/v2/go.sum | 84 ++++------- .../github.com/blevesearch/bleve/v2/index.go | 13 ++ .../bleve/v2/index/scorch/builder.go | 2 +- .../bleve/v2/index/scorch/introducer.go | 13 +- .../bleve/v2/index/scorch/merge.go | 42 +++--- .../bleve/v2/index/scorch/persister.go | 79 ++++++++++- .../bleve/v2/index/scorch/scorch.go | 19 +++ .../bleve/v2/index/scorch/snapshot_index.go | 48 +++++++ .../bleve/v2/index/upsidedown/upsidedown.go | 2 + .../blevesearch/bleve/v2/index_impl.go | 48 +++++++ .../blevesearch/bleve/v2/index_meta.go | 19 +++ .../blevesearch/bleve/v2/mapping.go | 10 ++ .../blevesearch/bleve/v2/mapping/document.go | 13 +- .../blevesearch/bleve/v2/mapping/field.go | 41 ++++++ .../github.com/blevesearch/bleve/v2/query.go | 9 ++ .../github.com/blevesearch/bleve/v2/search.go | 2 +- .../bleve/v2/search/collector/topn.go | 6 +- .../v2/search/facet/facet_builder_datetime.go | 32 ++--- .../v2/search/facet/facet_builder_numeric.go | 32 ++--- .../v2/search/facet/facet_builder_terms.go | 22 ++- .../bleve/v2/search/facets_builder.go | 113 +++++++++++---- .../highlight/fragmenter/simple/simple.go | 12 +- .../bleve/v2/search/query/ip_range.go | 84 +++++++++++ .../bleve/v2/search/query/query_string_lex.go | 14 +- .../blevesearch/bleve/v2/search/search.go | 2 +- .../v2/search/searcher/search_ip_range.go | 67 +++++++++ .../blevesearch/bleve/v2/search/sort.go | 3 +- .../blevesearch/bleve_index_api/directory.go | 23 +++ vendor/github.com/blevesearch/mmap-go/go.mod | 4 +- vendor/github.com/blevesearch/mmap-go/go.sum | 4 +- .../blevesearch/scorch_segment_api/v2/go.mod | 5 +- .../blevesearch/scorch_segment_api/v2/go.sum | 27 +--- vendor/github.com/blevesearch/vellum/go.mod | 5 +- vendor/github.com/blevesearch/vellum/go.sum | 5 +- .../github.com/blevesearch/zapx/v11/build.go | 56 ++++++-- vendor/github.com/blevesearch/zapx/v11/go.mod | 12 +- vendor/github.com/blevesearch/zapx/v11/go.sum | 59 ++------ .../blevesearch/zapx/v11/posting.go | 13 +- .../github.com/blevesearch/zapx/v12/build.go | 56 ++++++-- vendor/github.com/blevesearch/zapx/v12/go.mod | 12 +- vendor/github.com/blevesearch/zapx/v12/go.sum | 59 ++------ .../blevesearch/zapx/v12/posting.go | 13 +- .../github.com/blevesearch/zapx/v13/build.go | 56 ++++++-- vendor/github.com/blevesearch/zapx/v13/go.mod | 12 +- vendor/github.com/blevesearch/zapx/v13/go.sum | 59 ++------ .../blevesearch/zapx/v13/posting.go | 13 +- .../github.com/blevesearch/zapx/v14/build.go | 56 ++++++-- vendor/github.com/blevesearch/zapx/v14/go.mod | 12 +- vendor/github.com/blevesearch/zapx/v14/go.sum | 59 ++------ .../blevesearch/zapx/v14/posting.go | 13 +- .../github.com/blevesearch/zapx/v15/build.go | 56 ++++++-- vendor/github.com/blevesearch/zapx/v15/go.mod | 13 +- vendor/github.com/blevesearch/zapx/v15/go.sum | 56 ++------ .../blevesearch/zapx/v15/posting.go | 13 +- vendor/modules.txt | 23 ++- 70 files changed, 1262 insertions(+), 639 deletions(-) create mode 100644 vendor/github.com/blevesearch/bleve/v2/document/field_ip.go create mode 100644 vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go create mode 100644 vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go create mode 100644 vendor/github.com/blevesearch/bleve_index_api/directory.go diff --git a/go.mod b/go.mod index b608790e0c03..70eb74e99767 100644 --- a/go.mod +++ b/go.mod @@ -16,11 +16,10 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/ProtonMail/go-crypto v0.0.0-20210705153151-cc34b1f6908b // indirect github.com/PuerkitoBio/goquery v1.7.0 - github.com/RoaringBitmap/roaring v0.9.1 // indirect github.com/alecthomas/chroma v0.9.4 github.com/andybalholm/brotli v1.0.3 // indirect github.com/andybalholm/cascadia v1.2.0 // indirect - github.com/blevesearch/bleve/v2 v2.0.6 + github.com/blevesearch/bleve/v2 v2.3.0 github.com/boombuler/barcode v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect github.com/caddyserver/certmagic v0.14.1 diff --git a/go.sum b/go.sum index d817e9766907..7296a6574e33 100644 --- a/go.sum +++ b/go.sum @@ -62,7 +62,6 @@ github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzU github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -82,12 +81,9 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/RoaringBitmap/gocroaring v0.4.0/go.mod h1:NieMwz7ZqwU2DD73/vvYwv7r4eWBKuPVSXZIpsaMwCI= -github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76/go.mod h1:oM0MHmQ3nDsq609SS36p+oYbRi16+oVvU2Bw4Ipv0SE= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= -github.com/RoaringBitmap/roaring v0.9.1 h1:5PRizBmoN/PfV17nPNQou4dHQ7NcJi8FO/bihdYyCEM= -github.com/RoaringBitmap/roaring v0.9.1/go.mod h1:h1B7iIUOmnAeb5ytYMvnHJwxMc6LUrwBnzXWRuqTQUc= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -154,41 +150,43 @@ github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjL github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blevesearch/bleve/v2 v2.0.1/go.mod h1:OBP2Pktqik8vEiUlGhuWjYx7KiO4zD542+DHqICwM5w= -github.com/blevesearch/bleve/v2 v2.0.6 h1:2dV2S4pyUqQHftUFzM0htUCWC8MeRg2qsmgIvjnKlgU= -github.com/blevesearch/bleve/v2 v2.0.6/go.mod h1:UhqLjgDhN4mji6F1dL3fPghcqaBV6r6bXwKCdaBa3Is= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= +github.com/blevesearch/bleve/v2 v2.3.0 h1:5XKlSdpcjeJdE7n0FUEDeJRJwLuhPxq+k5n7h5UaJkg= +github.com/blevesearch/bleve/v2 v2.3.0/go.mod h1:egW/6gZEhM3oBvRjuHXGvGb92cKZ9867OqPZAmCG8MQ= github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= github.com/blevesearch/scorch_segment_api v1.0.0 h1:BUkCPWDg2gimTEyVDXf85I2buqqt4lh28uaVMiJsIYk= github.com/blevesearch/scorch_segment_api v1.0.0/go.mod h1:KgRYmlfYC27NeM6cXOHx8LBgq7jn0atpV8mVWoBKBng= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/blevesearch/zapx/v11 v11.1.10/go.mod h1:DTjbcBqrr/Uo82UBilDC8lEew42gN/OcIyiTNFtSijc= -github.com/blevesearch/zapx/v11 v11.2.1 h1:udluDHdr99gGSeL3vZLtJbML0OJ98mK1Peivtm5OYho= -github.com/blevesearch/zapx/v11 v11.2.1/go.mod h1:TBkJF5Qq0EwZbbBQmkW6/AQVSYwXXpp0xwtQ5wXHVMI= +github.com/blevesearch/zapx/v11 v11.3.2 h1:TDdcbaA0Yz3Y5zpTrpvyW1AeicqWTJL3g8D5g48RiHM= +github.com/blevesearch/zapx/v11 v11.3.2/go.mod h1:YzTfUm4kS3e8OmTXDHVV8OzC5MWPO/VPJZQgPNVb4Lc= github.com/blevesearch/zapx/v12 v12.1.10/go.mod h1:14NmKnPrnKAIyiEJM566k/Jk+FQpuiflT5d3uaaK3MI= -github.com/blevesearch/zapx/v12 v12.2.1 h1:nbeecR8M3dEcIIYfKDaSRpJ9E205E7BvjhVwf/l5ajI= -github.com/blevesearch/zapx/v12 v12.2.1/go.mod h1:sSXvgEs7MKqqDIRSpyFd6ZJUEVlhxuDB0d8/WT2WlgA= +github.com/blevesearch/zapx/v12 v12.3.2 h1:XB09XMg/3ibeIJRCm2zjkaVwrtAuk6c55YRSmVlwUDk= +github.com/blevesearch/zapx/v12 v12.3.2/go.mod h1:RMl6lOZqF+sTxKvhQDJ5yK2LT3Mu7E2p/jGdjAaiRxs= github.com/blevesearch/zapx/v13 v13.1.10/go.mod h1:YsVY6YGpTEAlJOMjdL7EsdBLvjWd8kPa2gwJDNpqLJo= -github.com/blevesearch/zapx/v13 v13.2.1 h1:6K797fvkurY6heEMPhyUlq3VULIpkD1sbBqqQUMFf4g= -github.com/blevesearch/zapx/v13 v13.2.1/go.mod h1:Fblcy4ykPy7XiaZ2svvpQaYgEqI+8vkdvMVx5zcawF4= +github.com/blevesearch/zapx/v13 v13.3.2 h1:mTvALh6oayreac07VRAv94FLvTHeSBM9sZ1gmVt0N2k= +github.com/blevesearch/zapx/v13 v13.3.2/go.mod h1:eppobNM35U4C22yDvTuxV9xPqo10pwfP/jugL4INWG4= github.com/blevesearch/zapx/v14 v14.1.10/go.mod h1:hsULl5eJSxs5NEfBsmeT9qrqdCP+/ecpVZKt60M4V64= -github.com/blevesearch/zapx/v14 v14.2.1 h1:V3RzDc7XZ51Kv9ZhhzMlHCSoY4+jxqy9VBqHxTqW4pg= -github.com/blevesearch/zapx/v14 v14.2.1/go.mod h1:veKtVCDzl4vvYeT5zULXEXqPR948uilzixzmmdtpCkU= +github.com/blevesearch/zapx/v14 v14.3.2 h1:oW36JVaZDzrzmBa1X5jdTIYzdhkOQnr/ie13Cb2X7MQ= +github.com/blevesearch/zapx/v14 v14.3.2/go.mod h1:zXNcVzukh0AvG57oUtT1T0ndi09H0kELNaNmekEy0jw= github.com/blevesearch/zapx/v15 v15.1.10/go.mod h1:4ypq25bwtSQKzwEF1UERyIhmGTbMT3brY/n4NC5gRnM= -github.com/blevesearch/zapx/v15 v15.2.1 h1:ZaqQiWLo0srtPvy3ozgpR9+Oabs3HQrF4uJM0HiKVBY= -github.com/blevesearch/zapx/v15 v15.2.1/go.mod h1:pUCN72ZJkVd7dU9lA4Fd8E3+fl5wv3JPpThk4FQ5bpA= +github.com/blevesearch/zapx/v15 v15.3.2 h1:OZNE4CQ9hQhnB21ySC7x2/9Q35U3WtRXLAh5L2gdCXc= +github.com/blevesearch/zapx/v15 v15.3.2/go.mod h1:C+f/97ZzTzK6vt/7sVlZdzZxKu+5+j4SrGCvr9dJzaY= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -240,6 +238,7 @@ github.com/couchbase/goutils v0.0.0-20201030094643-5e82bb967e67/go.mod h1:BQwMFl github.com/couchbase/goutils v0.0.0-20210118111533-e33d3ffb5401 h1:4KDlx3vjalrHD/EfsjCpV91HNX3JPaIqRtt83zZ7x+Y= github.com/couchbase/goutils v0.0.0-20210118111533-e33d3ffb5401/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= +github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -276,14 +275,12 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/editorconfig/editorconfig-core-go/v2 v2.4.2 h1:1lkDpSoAaFLrgYTVJ/eNCV+lkDSv/j9Wm0jcvDfVVEo= github.com/editorconfig/editorconfig-core-go/v2 v2.4.2/go.mod h1:IXeWRVO4LZRoNunhHh/oP6BQvTs94nB2pNvbw32l8tQ= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -1020,7 +1017,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6O github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1099,7 +1095,6 @@ github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -1540,7 +1535,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20200929161345-d7fc70abf50f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index 25cb8bf5c9a0..3a44a8dc0577 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -424,7 +424,7 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p } languagesFacet := result.Facets["languages"] - for _, term := range languagesFacet.Terms { + for _, term := range languagesFacet.Terms.Terms() { if len(term.Term) == 0 { continue } diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go index 260e1cbcef7e..80b7eecf7ef5 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -403,7 +403,7 @@ func (ac *arrayContainer) iorRun16(rc *runContainer16) container { var result container result = ac for _, run := range rc.iv { - result = result.iaddRange(int(run.start), int(run.start)+int(run.length)) + result = result.iaddRange(int(run.start), int(run.start)+int(run.length)+1) } return result } diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go index 342c7fd4844b..47bda7125dfc 100644 --- a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go +++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go @@ -33,15 +33,6 @@ main: s2 = x2.highlowcontainer.getKeyAtIndex(pos2) } else { c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - switch t := c1.(type) { - case *arrayContainer: - c1 = t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - c1 = t.toBitmapContainer() - } - } - answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) pos1++ pos2++ @@ -89,18 +80,7 @@ main: } s2 = x2.highlowcontainer.getKeyAtIndex(pos2) } else { - c1 := x1.highlowcontainer.getContainerAtIndex(pos1) - switch t := c1.(type) { - case *arrayContainer: - c1 = t.toBitmapContainer() - case *runContainer16: - if !t.isFull() { - c1 = t.toBitmapContainer() - } - case *bitmapContainer: - c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1) - } - + c1 := x1.highlowcontainer.getWritableContainerAtIndex(pos1) x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2)) x1.highlowcontainer.needCopyOnWrite[pos1] = false pos1++ diff --git a/vendor/github.com/RoaringBitmap/roaring/go.mod b/vendor/github.com/RoaringBitmap/roaring/go.mod index 88f2e925e772..d87811de2c60 100644 --- a/vendor/github.com/RoaringBitmap/roaring/go.mod +++ b/vendor/github.com/RoaringBitmap/roaring/go.mod @@ -3,8 +3,6 @@ module github.com/RoaringBitmap/roaring go 1.14 require ( - github.com/RoaringBitmap/gocroaring v0.4.0 - github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76 // indirect github.com/bits-and-blooms/bitset v1.2.0 github.com/mschoch/smat v0.2.0 github.com/stretchr/testify v1.4.0 diff --git a/vendor/github.com/RoaringBitmap/roaring/go.sum b/vendor/github.com/RoaringBitmap/roaring/go.sum index a6459b85418a..cd059d8860b8 100644 --- a/vendor/github.com/RoaringBitmap/roaring/go.sum +++ b/vendor/github.com/RoaringBitmap/roaring/go.sum @@ -1,7 +1,3 @@ -github.com/RoaringBitmap/gocroaring v0.4.0 h1:5nufXUgWpBEUNEJXw7926YAA58ZAQRpWPrQV1xCoSjc= -github.com/RoaringBitmap/gocroaring v0.4.0/go.mod h1:NieMwz7ZqwU2DD73/vvYwv7r4eWBKuPVSXZIpsaMwCI= -github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76 h1:ZYlhPbqQFU+AHfgtCdHGDTtRW1a8geZyiE8c6Q+Sl1s= -github.com/RoaringBitmap/real-roaring-datasets v0.0.0-20190726190000-eb7c87156f76/go.mod h1:oM0MHmQ3nDsq609SS36p+oYbRi16+oVvU2Bw4Ipv0SE= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go index 15332e4956c5..53068e4d933f 100644 --- a/vendor/github.com/RoaringBitmap/roaring/roaring.go +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -1552,3 +1552,27 @@ func (rb *Bitmap) Stats() Statistics { } return stats } + +func (rb *Bitmap) checkValidity() bool { + for _, c := range rb.highlowcontainer.containers { + + switch c.(type) { + case *arrayContainer: + if c.getCardinality() > arrayDefaultMaxSize { + fmt.Println("Array containers are limited to size ", arrayDefaultMaxSize) + return false + } + case *bitmapContainer: + if c.getCardinality() <= arrayDefaultMaxSize { + fmt.Println("Bitmaps would be more concise as an array!") + return false + } + case *runContainer16: + if c.getSizeInBytes() > minOfInt(bitmapContainerSizeInBytes(), arrayContainerSizeInBytes(c.getCardinality())) { + fmt.Println("Inefficient run container!") + return false + } + } + } + return true +} \ No newline at end of file diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go index 18a563e8d69c..221e173fe48c 100644 --- a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -297,8 +297,9 @@ func (ra *roaringArray) frozenView(buf []byte) error { iBitset++ case 2: containers[i] = &arrays[iArray] - arrays[iArray].content = arraysArena[:counts[i]+1] - arraysArena = arraysArena[counts[i]+1:] + sz := int(counts[i])+1 + arrays[iArray].content = arraysArena[:sz] + arraysArena = arraysArena[sz:] iArray++ case 3: containers[i] = &runs[iRun] @@ -401,7 +402,7 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) { copy(arraysArena, v.content) arraysArena = arraysArena[len(v.content):] elems := len(v.content) - counts[i] = uint16(elems)-1 + counts[i] = uint16(elems-1) types[i] = 2 case *runContainer16: copy(runsArena, v.iv) diff --git a/vendor/github.com/blevesearch/bleve/v2/.gitignore b/vendor/github.com/blevesearch/bleve/v2/.gitignore index ab7a1e21a680..7512de770ffe 100644 --- a/vendor/github.com/blevesearch/bleve/v2/.gitignore +++ b/vendor/github.com/blevesearch/bleve/v2/.gitignore @@ -17,3 +17,4 @@ vendor/** /search/query/y.output *.test tags +go.sum diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/type.go b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go index 589cc1ca6a28..1fa26308cd0b 100644 --- a/vendor/github.com/blevesearch/bleve/v2/analysis/type.go +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/type.go @@ -34,6 +34,7 @@ const ( Single Double Boolean + IP ) // Token represents one occurrence of a term at a particular location in a diff --git a/vendor/github.com/blevesearch/bleve/v2/config_app.go b/vendor/github.com/blevesearch/bleve/v2/config_app.go index 112d0b600db7..60b1db3e89fe 100644 --- a/vendor/github.com/blevesearch/bleve/v2/config_app.go +++ b/vendor/github.com/blevesearch/bleve/v2/config_app.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build appengine || appenginevm // +build appengine appenginevm package bleve diff --git a/vendor/github.com/blevesearch/bleve/v2/config_disk.go b/vendor/github.com/blevesearch/bleve/v2/config_disk.go index ccfd6da8c031..a9ab1e41c90d 100644 --- a/vendor/github.com/blevesearch/bleve/v2/config_disk.go +++ b/vendor/github.com/blevesearch/bleve/v2/config_disk.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !appengine && !appenginevm // +build !appengine,!appenginevm package bleve diff --git a/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go b/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go new file mode 100644 index 000000000000..1e5be5006a92 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/document/field_ip.go @@ -0,0 +1,132 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + "net" + "reflect" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/size" + index "github.com/blevesearch/bleve_index_api" +) + +var reflectStaticSizeIPField int + +func init() { + var f IPField + reflectStaticSizeIPField = int(reflect.TypeOf(f).Size()) +} + +const DefaultIPIndexingOptions = index.StoreField | index.IndexField | index.DocValues | index.IncludeTermVectors + +type IPField struct { + name string + arrayPositions []uint64 + options index.FieldIndexingOptions + value net.IP + numPlainTextBytes uint64 + length int + frequencies index.TokenFrequencies +} + +func (b *IPField) Size() int { + return reflectStaticSizeIPField + size.SizeOfPtr + + len(b.name) + + len(b.arrayPositions)*size.SizeOfUint64 + + len(b.value) +} + +func (b *IPField) Name() string { + return b.name +} + +func (b *IPField) ArrayPositions() []uint64 { + return b.arrayPositions +} + +func (b *IPField) Options() index.FieldIndexingOptions { + return b.options +} + +func (n *IPField) EncodedFieldType() byte { + return 'i' +} + +func (n *IPField) AnalyzedLength() int { + return n.length +} + +func (n *IPField) AnalyzedTokenFrequencies() index.TokenFrequencies { + return n.frequencies +} + +func (b *IPField) Analyze() { + + tokens := analysis.TokenStream{ + &analysis.Token{ + Start: 0, + End: len(b.value), + Term: b.value, + Position: 1, + Type: analysis.IP, + }, + } + b.length = 1 + b.frequencies = analysis.TokenFrequency(tokens, b.arrayPositions, b.options) +} + +func (b *IPField) Value() []byte { + return b.value +} + +func (b *IPField) IP() (net.IP, error) { + return net.IP(b.value), nil +} + +func (b *IPField) GoString() string { + return fmt.Sprintf("&document.IPField{Name:%s, Options: %s, Value: %s}", b.name, b.options, net.IP(b.value)) +} + +func (b *IPField) NumPlainTextBytes() uint64 { + return b.numPlainTextBytes +} + +func NewIPFieldFromBytes(name string, arrayPositions []uint64, value []byte) *IPField { + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewIPField(name string, arrayPositions []uint64, v net.IP) *IPField { + return NewIPFieldWithIndexingOptions(name, arrayPositions, v, DefaultIPIndexingOptions) +} + +func NewIPFieldWithIndexingOptions(name string, arrayPositions []uint64, b net.IP, options index.FieldIndexingOptions) *IPField { + v := b.To16() + + return &IPField{ + name: name, + arrayPositions: arrayPositions, + value: v, + options: options, + numPlainTextBytes: net.IPv6len, + } +} diff --git a/vendor/github.com/blevesearch/bleve/v2/go.mod b/vendor/github.com/blevesearch/bleve/v2/go.mod index a7545710be50..1bd6b0f1d601 100644 --- a/vendor/github.com/blevesearch/bleve/v2/go.mod +++ b/vendor/github.com/blevesearch/bleve/v2/go.mod @@ -3,21 +3,21 @@ module github.com/blevesearch/bleve/v2 go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 + github.com/RoaringBitmap/roaring v0.9.4 github.com/bits-and-blooms/bitset v1.2.0 - github.com/blevesearch/bleve_index_api v1.0.0 + github.com/blevesearch/bleve_index_api v1.0.1 github.com/blevesearch/go-porterstemmer v1.0.3 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 github.com/blevesearch/segment v0.9.0 github.com/blevesearch/snowballstem v0.9.0 github.com/blevesearch/upsidedown_store_api v1.0.1 - github.com/blevesearch/vellum v1.0.5 - github.com/blevesearch/zapx/v11 v11.2.1 - github.com/blevesearch/zapx/v12 v12.2.1 - github.com/blevesearch/zapx/v13 v13.2.1 - github.com/blevesearch/zapx/v14 v14.2.1 - github.com/blevesearch/zapx/v15 v15.2.1 - github.com/couchbase/moss v0.1.0 + github.com/blevesearch/vellum v1.0.7 + github.com/blevesearch/zapx/v11 v11.3.2 + github.com/blevesearch/zapx/v12 v12.3.2 + github.com/blevesearch/zapx/v13 v13.3.2 + github.com/blevesearch/zapx/v14 v14.3.2 + github.com/blevesearch/zapx/v15 v15.3.2 + github.com/couchbase/moss v0.2.0 github.com/golang/protobuf v1.3.2 github.com/kljensen/snowball v0.6.0 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 @@ -25,5 +25,5 @@ require ( github.com/steveyen/gtreap v0.1.0 github.com/syndtr/goleveldb v1.0.0 go.etcd.io/bbolt v1.3.5 - golang.org/x/text v0.3.0 + golang.org/x/text v0.3.7 ) diff --git a/vendor/github.com/blevesearch/bleve/v2/go.sum b/vendor/github.com/blevesearch/bleve/v2/go.sum index 673026008192..b11b5f434de5 100644 --- a/vendor/github.com/blevesearch/bleve/v2/go.sum +++ b/vendor/github.com/blevesearch/bleve/v2/go.sum @@ -1,73 +1,65 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= -github.com/blevesearch/zapx/v11 v11.2.1 h1:udluDHdr99gGSeL3vZLtJbML0OJ98mK1Peivtm5OYho= -github.com/blevesearch/zapx/v11 v11.2.1/go.mod h1:TBkJF5Qq0EwZbbBQmkW6/AQVSYwXXpp0xwtQ5wXHVMI= -github.com/blevesearch/zapx/v12 v12.2.1 h1:nbeecR8M3dEcIIYfKDaSRpJ9E205E7BvjhVwf/l5ajI= -github.com/blevesearch/zapx/v12 v12.2.1/go.mod h1:sSXvgEs7MKqqDIRSpyFd6ZJUEVlhxuDB0d8/WT2WlgA= -github.com/blevesearch/zapx/v13 v13.2.1 h1:6K797fvkurY6heEMPhyUlq3VULIpkD1sbBqqQUMFf4g= -github.com/blevesearch/zapx/v13 v13.2.1/go.mod h1:Fblcy4ykPy7XiaZ2svvpQaYgEqI+8vkdvMVx5zcawF4= -github.com/blevesearch/zapx/v14 v14.2.1 h1:V3RzDc7XZ51Kv9ZhhzMlHCSoY4+jxqy9VBqHxTqW4pg= -github.com/blevesearch/zapx/v14 v14.2.1/go.mod h1:veKtVCDzl4vvYeT5zULXEXqPR948uilzixzmmdtpCkU= -github.com/blevesearch/zapx/v15 v15.2.1 h1:ZaqQiWLo0srtPvy3ozgpR9+Oabs3HQrF4uJM0HiKVBY= -github.com/blevesearch/zapx/v15 v15.2.1/go.mod h1:pUCN72ZJkVd7dU9lA4Fd8E3+fl5wv3JPpThk4FQ5bpA= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= +github.com/blevesearch/zapx/v11 v11.3.2 h1:TDdcbaA0Yz3Y5zpTrpvyW1AeicqWTJL3g8D5g48RiHM= +github.com/blevesearch/zapx/v11 v11.3.2/go.mod h1:YzTfUm4kS3e8OmTXDHVV8OzC5MWPO/VPJZQgPNVb4Lc= +github.com/blevesearch/zapx/v12 v12.3.2 h1:XB09XMg/3ibeIJRCm2zjkaVwrtAuk6c55YRSmVlwUDk= +github.com/blevesearch/zapx/v12 v12.3.2/go.mod h1:RMl6lOZqF+sTxKvhQDJ5yK2LT3Mu7E2p/jGdjAaiRxs= +github.com/blevesearch/zapx/v13 v13.3.2 h1:mTvALh6oayreac07VRAv94FLvTHeSBM9sZ1gmVt0N2k= +github.com/blevesearch/zapx/v13 v13.3.2/go.mod h1:eppobNM35U4C22yDvTuxV9xPqo10pwfP/jugL4INWG4= +github.com/blevesearch/zapx/v14 v14.3.2 h1:oW36JVaZDzrzmBa1X5jdTIYzdhkOQnr/ie13Cb2X7MQ= +github.com/blevesearch/zapx/v14 v14.3.2/go.mod h1:zXNcVzukh0AvG57oUtT1T0ndi09H0kELNaNmekEy0jw= +github.com/blevesearch/zapx/v15 v15.3.2 h1:OZNE4CQ9hQhnB21ySC7x2/9Q35U3WtRXLAh5L2gdCXc= +github.com/blevesearch/zapx/v15 v15.3.2/go.mod h1:C+f/97ZzTzK6vt/7sVlZdzZxKu+5+j4SrGCvr9dJzaY= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= -github.com/couchbase/moss v0.1.0 h1:HCL+xxHUwmOaL44kMM/gU08OW6QGCui1WVFO58bjhNI= -github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= +github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= +github.com/couchbase/moss v0.2.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kljensen/snowball v0.6.0 h1:6DZLCcZeL0cLfodx+Md4/OLC6b/bfurWUOUGs1ydfOU= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -76,12 +68,10 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -91,55 +81,33 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/vendor/github.com/blevesearch/bleve/v2/index.go b/vendor/github.com/blevesearch/bleve/v2/index.go index e08271e511ed..e297122fac72 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index.go +++ b/vendor/github.com/blevesearch/bleve/v2/index.go @@ -16,6 +16,7 @@ package bleve import ( "context" + "github.com/blevesearch/bleve/v2/index/upsidedown" "github.com/blevesearch/bleve/v2/document" @@ -306,3 +307,15 @@ type Builder interface { func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) { return newBuilder(path, mapping, config) } + +// IndexCopyable is an index which supports an online copy operation +// of the index. +type IndexCopyable interface { + // CopyTo creates a fully functional copy of the index at the + // specified destination directory implementation. + CopyTo(d index.Directory) error +} + +// FileSystemDirectory is the default implementation for the +// index.Directory interface. +type FileSystemDirectory string diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go index 3311bd02e4bb..5e64230c51b3 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/builder.go @@ -304,7 +304,7 @@ func (o *Builder) Close() error { } // fill the root bolt with this fake index snapshot - _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin) + _, _, err = prepareBoltSnapshot(is, tx, o.path, o.segPlugin, nil) if err != nil { _ = tx.Rollback() _ = rootBolt.Close() diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go index 8516d411dbbf..74373c4af3df 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/introducer.go @@ -46,6 +46,17 @@ type epochWatcher struct { } func (s *Scorch) introducerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "introducer", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() + var epochWatchers []*epochWatcher OUTER: for { @@ -88,8 +99,6 @@ OUTER: } epochWatchers = epochWatchersNext } - - s.asyncTasks.Done() } func (s *Scorch) introduceSegment(next *segmentIntroduction) error { diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go index 1e269afc6310..422527c6f6e8 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/merge.go @@ -29,12 +29,22 @@ import ( ) func (s *Scorch) mergerLoop() { + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "merger", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() + var lastEpochMergePlanned uint64 var ctrlMsg *mergerCtrl mergePlannerOptions, err := s.parseMergePlannerOptions() if err != nil { s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err)) - s.asyncTasks.Done() return } ctrlMsgDflt := &mergerCtrl{ctx: context.Background(), @@ -130,8 +140,6 @@ OUTER: atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1) } - - s.asyncTasks.Done() } type mergerCtrl struct { @@ -209,32 +217,32 @@ func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, } type closeChWrapper struct { - ch1 chan struct{} - ctx context.Context - closeCh chan struct{} + ch1 chan struct{} + ctx context.Context + closeCh chan struct{} + cancelCh chan struct{} } func newCloseChWrapper(ch1 chan struct{}, ctx context.Context) *closeChWrapper { - return &closeChWrapper{ch1: ch1, - ctx: ctx, - closeCh: make(chan struct{})} + return &closeChWrapper{ + ch1: ch1, + ctx: ctx, + closeCh: make(chan struct{}), + cancelCh: make(chan struct{}), + } } func (w *closeChWrapper) close() { - select { - case <-w.closeCh: - default: - close(w.closeCh) - } + close(w.closeCh) } func (w *closeChWrapper) listen() { select { case <-w.ch1: - w.close() + close(w.cancelCh) case <-w.ctx.Done(): - w.close() + close(w.cancelCh) case <-w.closeCh: } } @@ -320,7 +328,7 @@ func (s *Scorch) planMergeAtSnapshot(ctx context.Context, atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) newDocNums, _, err := s.segPlugin.Merge(segmentsToMerge, docsToDrop, path, - cw.closeCh, s) + cw.cancelCh, s) atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go index 00c92429e440..670f0f820c96 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/persister.go @@ -19,6 +19,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "io" "io/ioutil" "log" "math" @@ -82,7 +83,16 @@ type persisterOptions struct { type notificationChan chan struct{} func (s *Scorch) persisterLoop() { - defer s.asyncTasks.Done() + defer func() { + if r := recover(); r != nil { + s.fireAsyncError(&AsyncPanicError{ + Source: "persister", + Path: s.path, + }) + } + + s.asyncTasks.Done() + }() var persistWatchers []*epochWatcher var lastPersistedEpoch, lastMergedEpoch uint64 @@ -427,8 +437,59 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) ( return true, nil } +func copyToDirectory(srcPath string, d index.Directory) (int64, error) { + if d == nil { + return 0, nil + } + + dest, err := d.GetWriter(filepath.Join("store", filepath.Base(srcPath))) + if err != nil { + return 0, fmt.Errorf("GetWriter err: %v", err) + } + + sourceFileStat, err := os.Stat(srcPath) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", srcPath) + } + + source, err := os.Open(srcPath) + if err != nil { + return 0, err + } + defer source.Close() + defer dest.Close() + return io.Copy(dest, source) +} + +func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory, + path string) error { + if d == nil { + return seg.Persist(path) + } + + sg, ok := seg.(io.WriterTo) + if !ok { + return fmt.Errorf("no io.WriterTo segment implementation found") + } + + w, err := d.GetWriter(filepath.Join("store", filepath.Base(path))) + if err != nil { + return err + } + + _, err = sg.WriteTo(w) + w.Close() + + return err +} + func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, - segPlugin SegmentPlugin) ([]string, map[uint64]string, error) { + segPlugin SegmentPlugin, d index.Directory) ( + []string, map[uint64]string, error) { snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) if err != nil { return nil, nil, err @@ -481,7 +542,11 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, switch seg := segmentSnapshot.segment.(type) { case segment.PersistedSegment: segPath := seg.Path() - filename := strings.TrimPrefix(segPath, path+string(os.PathSeparator)) + _, err = copyToDirectory(segPath, d) + if err != nil { + return nil, nil, fmt.Errorf("segment: %s copy err: %v", segPath, err) + } + filename := filepath.Base(segPath) err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) if err != nil { return nil, nil, err @@ -490,10 +555,10 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, case segment.UnpersistedSegment: // need to persist this to disk filename := zapFileName(segmentSnapshot.id) - path := path + string(os.PathSeparator) + filename - err = seg.Persist(path) + path := filepath.Join(path, filename) + err := persistToDirectory(seg, d, path) if err != nil { - return nil, nil, fmt.Errorf("error persisting segment: %v", err) + return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err) } newSegmentPaths[segmentSnapshot.id] = path err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) @@ -534,7 +599,7 @@ func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { } }() - filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin) + filenames, newSegmentPaths, err := prepareBoltSnapshot(snapshot, tx, s.path, s.segPlugin, nil) if err != nil { return err } diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go index fa1aaebbfef1..abad722eccdf 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/scorch.go @@ -75,6 +75,16 @@ type Scorch struct { segPlugin SegmentPlugin } +// AsyncPanicError is passed to scorch asyncErrorHandler when panic occurs in scorch background process +type AsyncPanicError struct { + Source string + Path string +} + +func (e *AsyncPanicError) Error() string { + return fmt.Sprintf("%s panic when processing %s", e.Source, e.Path) +} + type internalStats struct { persistEpoch uint64 persistSnapshotSize uint64 @@ -202,6 +212,15 @@ func (s *Scorch) openBolt() error { var rootBoltOpt = *bolt.DefaultOptions if s.readOnly { rootBoltOpt.ReadOnly = true + rootBoltOpt.OpenFile = func(path string, flag int, mode os.FileMode) (*os.File, error) { + // Bolt appends an O_CREATE flag regardless. + // See - https://github.com/etcd-io/bbolt/blob/v1.3.5/db.go#L210 + // Use os.O_RDONLY only if path exists (#1623) + if _, err := os.Stat(path); os.IsNotExist(err) { + return os.OpenFile(path, flag, mode) + } + return os.OpenFile(path, os.O_RDONLY, mode) + } } else { if s.path != "" { err := os.MkdirAll(s.path, 0700) diff --git a/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go index ac2f34412070..ba8a771269a0 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/scorch/snapshot_index.go @@ -18,6 +18,8 @@ import ( "container/heap" "encoding/binary" "fmt" + "os" + "path/filepath" "reflect" "sort" "sync" @@ -29,6 +31,7 @@ import ( segment "github.com/blevesearch/scorch_segment_api/v2" "github.com/blevesearch/vellum" lev "github.com/blevesearch/vellum/levenshtein" + bolt "go.etcd.io/bbolt" ) // re usable, threadsafe levenshtein builders @@ -426,6 +429,8 @@ func (i *IndexSnapshot) Document(id string) (rv index.Document, err error) { rvd.AddField(document.NewTextField(name, arrayPos, value)) case 'n': rvd.AddField(document.NewNumericFieldFromBytes(name, arrayPos, value)) + case 'i': + rvd.AddField(document.NewIPFieldFromBytes(name, arrayPos, value)) case 'd': rvd.AddField(document.NewDateTimeFieldFromBytes(name, arrayPos, value)) case 'b': @@ -762,3 +767,46 @@ OUTER: } return rv } + +func (i *IndexSnapshot) CopyTo(d index.Directory) error { + // get the root bolt file. + w, err := d.GetWriter(filepath.Join("store", "root.bolt")) + if err != nil || w == nil { + return fmt.Errorf("failed to create the root.bolt file, err: %v", err) + } + rootFile, ok := w.(*os.File) + if !ok { + return fmt.Errorf("invalid root.bolt file found") + } + + copyBolt, err := bolt.Open(rootFile.Name(), 0600, nil) + if err != nil { + return err + } + defer func() { + w.Close() + if cerr := copyBolt.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // start a write transaction + tx, err := copyBolt.Begin(true) + if err != nil { + return err + } + + _, _, err = prepareBoltSnapshot(i, tx, "", i.parent.segPlugin, d) + if err != nil { + _ = tx.Rollback() + return fmt.Errorf("error backing up index snapshot: %v", err) + } + + // commit bolt data + err = tx.Commit() + if err != nil { + return fmt.Errorf("error commit tx to backup root bolt: %v", err) + } + + return copyBolt.Sync() +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go index d67ee03fb021..34c331942b28 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/v2/index/upsidedown/upsidedown.go @@ -727,6 +727,8 @@ func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document return document.NewBooleanFieldFromBytes(name, pos, value) case 'g': return document.NewGeoPointFieldFromBytes(name, pos, value) + case 'i': + return document.NewIPFieldFromBytes(name, pos, value) } return nil } diff --git a/vendor/github.com/blevesearch/bleve/v2/index_impl.go b/vendor/github.com/blevesearch/bleve/v2/index_impl.go index 879a366347ca..8a9cfd3b4233 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_impl.go @@ -18,7 +18,9 @@ import ( "context" "encoding/json" "fmt" + "io" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -910,3 +912,49 @@ func (m *searchHitSorter) Less(i, j int) bool { c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) return c < 0 } + +func (i *indexImpl) CopyTo(d index.Directory) (err error) { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + indexReader, err := i.i.Reader() + if err != nil { + return err + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + irc, ok := indexReader.(IndexCopyable) + if !ok { + return fmt.Errorf("index implementation does not support copy") + } + + err = irc.CopyTo(d) + if err != nil { + return fmt.Errorf("error copying index metadata: %v", err) + } + + // copy the metadata + return i.meta.CopyTo(d) +} + +func (f FileSystemDirectory) GetWriter(filePath string) (io.WriteCloser, + error) { + dir, file := filepath.Split(filePath) + if dir != "" { + err := os.MkdirAll(filepath.Join(string(f), dir), os.ModePerm) + if err != nil { + return nil, err + } + } + + return os.OpenFile(filepath.Join(string(f), dir, file), + os.O_RDWR|os.O_CREATE, 0600) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/index_meta.go b/vendor/github.com/blevesearch/bleve/v2/index_meta.go index fe0ddebac37d..711e0c1bd6d7 100644 --- a/vendor/github.com/blevesearch/bleve/v2/index_meta.go +++ b/vendor/github.com/blevesearch/bleve/v2/index_meta.go @@ -16,11 +16,13 @@ package bleve import ( "encoding/json" + "fmt" "io/ioutil" "os" "path/filepath" "github.com/blevesearch/bleve/v2/index/upsidedown" + index "github.com/blevesearch/bleve_index_api" ) const metaFilename = "index_meta.json" @@ -92,6 +94,23 @@ func (i *indexMeta) Save(path string) (err error) { return nil } +func (i *indexMeta) CopyTo(d index.Directory) (err error) { + metaBytes, err := json.Marshal(i) + if err != nil { + return err + } + + w, err := d.GetWriter(metaFilename) + if w == nil || err != nil { + return fmt.Errorf("invalid writer for file: %s, err: %v", + metaFilename, err) + } + defer w.Close() + + _, err = w.Write(metaBytes) + return err +} + func indexMetaPath(path string) string { return filepath.Join(path, metaFilename) } diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping.go b/vendor/github.com/blevesearch/bleve/v2/mapping.go index 54753073a6c6..c8db45df94f7 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping.go @@ -45,6 +45,12 @@ func NewTextFieldMapping() *mapping.FieldMapping { return mapping.NewTextFieldMapping() } +// NewKeywordFieldMapping returns a field mapping for text using the keyword +// analyzer, which essentially doesn't apply any specific text analysis. +func NewKeywordFieldMapping() *mapping.FieldMapping { + return mapping.NewKeywordFieldMapping() +} + // NewNumericFieldMapping returns a default field mapping for numbers func NewNumericFieldMapping() *mapping.FieldMapping { return mapping.NewNumericFieldMapping() @@ -63,3 +69,7 @@ func NewBooleanFieldMapping() *mapping.FieldMapping { func NewGeoPointFieldMapping() *mapping.FieldMapping { return mapping.NewGeoPointFieldMapping() } + +func NewIPFieldMapping() *mapping.FieldMapping { + return mapping.NewIPFieldMapping() +} diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/document.go b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go index 44911de77c3c..ae881dba615d 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/document.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/document.go @@ -18,6 +18,7 @@ import ( "encoding" "encoding/json" "fmt" + "net" "reflect" "time" @@ -76,7 +77,7 @@ func (dm *DocumentMapping) Validate(cache *registry.Cache) error { } } switch field.Type { - case "text", "datetime", "number", "boolean", "geopoint": + case "text", "datetime", "number", "boolean", "geopoint", "IP": default: return fmt.Errorf("unknown field type: '%s'", field.Type) } @@ -517,8 +518,14 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, case reflect.Map, reflect.Slice: if subDocMapping != nil { for _, fieldMapping := range subDocMapping.Fields { - if fieldMapping.Type == "geopoint" { + switch fieldMapping.Type { + case "geopoint": fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + case "IP": + ip, ok := property.(net.IP) + if ok { + fieldMapping.processIP(ip, pathString, path, indexes, context) + } } } } @@ -528,7 +535,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, switch property := property.(type) { case encoding.TextMarshaler: // ONLY process TextMarshaler if there is an explicit mapping - // AND all of the fiels are of type text + // AND all of the fields are of type text // OTHERWISE process field without TextMarshaler if subDocMapping != nil { allFieldsText := true diff --git a/vendor/github.com/blevesearch/bleve/v2/mapping/field.go b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go index 7ad1744d4e83..6e4f5290054e 100644 --- a/vendor/github.com/blevesearch/bleve/v2/mapping/field.go +++ b/vendor/github.com/blevesearch/bleve/v2/mapping/field.go @@ -17,8 +17,10 @@ package mapping import ( "encoding/json" "fmt" + "net" "time" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" index "github.com/blevesearch/bleve_index_api" "github.com/blevesearch/bleve/v2/analysis" @@ -89,6 +91,19 @@ func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { return rv } +// NewKeyworFieldMapping returns a default field mapping for text with analyzer "keyword". +func NewKeywordFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Analyzer: keyword.Name, + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + DocValues: true, + } +} + // NewNumericFieldMapping returns a default field mapping for numbers func NewNumericFieldMapping() *FieldMapping { return &FieldMapping{ @@ -157,6 +172,16 @@ func NewGeoPointFieldMapping() *FieldMapping { } } +// NewIPFieldMapping returns a default field mapping for IP points +func NewIPFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "IP", + Store: true, + Index: true, + IncludeInAll: true, + } +} + // Options returns the indexing options for this field. func (fm *FieldMapping) Options() index.FieldIndexingOptions { var rv index.FieldIndexingOptions @@ -201,6 +226,11 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str fm.processTime(parsedDateTime, pathString, path, indexes, context) } } + } else if fm.Type == "IP" { + ip := net.ParseIP(propertyValueString) + if ip != nil { + fm.processIP(ip, pathString, path, indexes, context) + } } } @@ -261,6 +291,17 @@ func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pat } } +func (fm *FieldMapping) processIP(ip net.IP, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewIPFieldWithIndexingOptions(fieldName, indexes, ip, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } +} + func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer { analyzerName := fm.Analyzer if analyzerName == "" { diff --git a/vendor/github.com/blevesearch/bleve/v2/query.go b/vendor/github.com/blevesearch/bleve/v2/query.go index 91d2d5f421a0..4f1f136ce31b 100644 --- a/vendor/github.com/blevesearch/bleve/v2/query.go +++ b/vendor/github.com/blevesearch/bleve/v2/query.go @@ -216,3 +216,12 @@ func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightL func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { return query.NewGeoDistanceQuery(lon, lat, distance) } + +// NewIPRangeQuery creates a new Query for matching IP addresses. +// If the argument is in CIDR format, then the query will match all +// IP addresses in the network specified. If the argument is an IP address, +// then the query will return documents which contain that IP. +// Both ipv4 and ipv6 are supported. +func NewIPRangeQuery(cidr string) *query.IPRangeQuery { + return query.NewIPRangeQuery(cidr) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search.go b/vendor/github.com/blevesearch/bleve/v2/search.go index 7397f566ea5a..2276aa011451 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search.go @@ -543,7 +543,7 @@ func (sr *SearchResult) String() string { rv += fmt.Sprintf("Facets:\n") for fn, f := range sr.Facets { rv += fmt.Sprintf("%s(%d)\n", fn, f.Total) - for _, t := range f.Terms { + for _, t := range f.Terms.Terms() { rv += fmt.Sprintf("\t%s(%d)\n", t.Term, t.Count) } if f.Other != 0 { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go index aa1d65bd172b..13d31e06f3cd 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/collector/topn.go @@ -83,7 +83,7 @@ func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector return newTopNCollector(size, skip, sort) } -// NewTopNCollector builds a collector to find the top 'size' hits +// NewTopNCollectorAfter builds a collector to find the top 'size' hits // skipping over the first 'skip' hits // ordering hits by the provided sort order func NewTopNCollectorAfter(size int, sort search.SortOrder, after []string) *TopNCollector { @@ -235,9 +235,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, // compute search duration hc.took = time.Since(startTime) - if err != nil { - return err - } + // finalize actual results err = hc.finalizeResults(reader) if err != nil { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go index a316ee4da86a..ff5167f21b16 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_datetime.go @@ -87,23 +87,21 @@ func (fb *DateTimeFacetBuilder) Field() string { return fb.field } -func (fb *DateTimeFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - t := time.Unix(0, i64) - - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { - if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { - fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 - fb.total++ - } +func (fb *DateTimeFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + t := time.Unix(0, i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ } } } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go index 6d0c6c9d135e..f19634d7b6df 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_numeric.go @@ -86,23 +86,21 @@ func (fb *NumericFacetBuilder) Field() string { return fb.field } -func (fb *NumericFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - // only consider the values which are shifted 0 - prefixCoded := numeric.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - f64 := numeric.Int64ToFloat64(i64) - - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { - if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { - fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 - fb.total++ - } +func (fb *NumericFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + f64 := numeric.Int64ToFloat64(i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ } } } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go index 1b378db7cd65..c5a1c831815d 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facet/facet_builder_terms.go @@ -62,12 +62,10 @@ func (fb *TermsFacetBuilder) Field() string { return fb.field } -func (fb *TermsFacetBuilder) UpdateVisitor(field string, term []byte) { - if field == fb.field { - fb.sawValue = true - fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1 - fb.total++ - } +func (fb *TermsFacetBuilder) UpdateVisitor(term []byte) { + fb.sawValue = true + fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1 + fb.total++ } func (fb *TermsFacetBuilder) StartDoc() { @@ -87,7 +85,7 @@ func (fb *TermsFacetBuilder) Result() *search.FacetResult { Missing: fb.missing, } - rv.Terms = make([]*search.TermFacet, 0, len(fb.termsCount)) + rv.Terms = &search.TermFacets{} for term, count := range fb.termsCount { tf := &search.TermFacet{ @@ -95,20 +93,20 @@ func (fb *TermsFacetBuilder) Result() *search.FacetResult { Count: count, } - rv.Terms = append(rv.Terms, tf) + rv.Terms.Add(tf) } sort.Sort(rv.Terms) // we now have the list of the top N facets trimTopN := fb.size - if trimTopN > len(rv.Terms) { - trimTopN = len(rv.Terms) + if trimTopN > rv.Terms.Len() { + trimTopN = rv.Terms.Len() } - rv.Terms = rv.Terms[:trimTopN] + rv.Terms.TrimToTopN(trimTopN) notOther := 0 - for _, tf := range rv.Terms { + for _, tf := range rv.Terms.Terms() { notOther += tf.Count } rv.Other = fb.total - notOther diff --git a/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go index 9822257f7e3e..e6afdf484c42 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/facets_builder.go @@ -15,6 +15,7 @@ package search import ( + "encoding/json" "reflect" "sort" @@ -43,7 +44,7 @@ func init() { type FacetBuilder interface { StartDoc() - UpdateVisitor(field string, term []byte) + UpdateVisitor(term []byte) EndDoc() Result() *FacetResult @@ -53,10 +54,11 @@ type FacetBuilder interface { } type FacetsBuilder struct { - indexReader index.IndexReader - facetNames []string - facets []FacetBuilder - fields []string + indexReader index.IndexReader + facetNames []string + facets []FacetBuilder + facetsByField map[string][]FacetBuilder + fields []string } func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { @@ -80,8 +82,13 @@ func (fb *FacetsBuilder) Size() int { } func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { + if fb.facetsByField == nil { + fb.facetsByField = map[string][]FacetBuilder{} + } + fb.facetNames = append(fb.facetNames, name) fb.facets = append(fb.facets, facetBuilder) + fb.facetsByField[facetBuilder.Field()] = append(fb.facetsByField[facetBuilder.Field()], facetBuilder) fb.fields = append(fb.fields, facetBuilder.Field()) } @@ -102,8 +109,10 @@ func (fb *FacetsBuilder) EndDoc() { } func (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) { - for _, facetBuilder := range fb.facets { - facetBuilder.UpdateVisitor(field, term) + if facetBuilders, ok := fb.facetsByField[field]; ok { + for _, facetBuilder := range facetBuilders { + facetBuilder.UpdateVisitor(term) + } } } @@ -112,27 +121,73 @@ type TermFacet struct { Count int `json:"count"` } -type TermFacets []*TermFacet +type TermFacets struct { + termFacets []*TermFacet + termLookup map[string]*TermFacet +} + +func (tf *TermFacets) Terms() []*TermFacet { + return tf.termFacets +} -func (tf TermFacets) Add(termFacet *TermFacet) TermFacets { - for _, existingTerm := range tf { - if termFacet.Term == existingTerm.Term { - existingTerm.Count += termFacet.Count - return tf +func (tf *TermFacets) TrimToTopN(n int) { + tf.termFacets = tf.termFacets[:n] +} + +func (tf *TermFacets) Add(termFacets ...*TermFacet) { + for _, termFacet := range termFacets { + if tf.termLookup == nil { + tf.termLookup = map[string]*TermFacet{} + } + + if term, ok := tf.termLookup[termFacet.Term]; ok { + term.Count += termFacet.Count + return } + + // if we got here it wasn't already in the existing terms + tf.termFacets = append(tf.termFacets, termFacet) + tf.termLookup[termFacet.Term] = termFacet } - // if we got here it wasn't already in the existing terms - tf = append(tf, termFacet) - return tf } -func (tf TermFacets) Len() int { return len(tf) } -func (tf TermFacets) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } -func (tf TermFacets) Less(i, j int) bool { - if tf[i].Count == tf[j].Count { - return tf[i].Term < tf[j].Term +func (tf *TermFacets) Len() int { + // Handle case where *TermFacets is not fully initialized in index_impl.go.init() + if tf == nil { + return 0 } - return tf[i].Count > tf[j].Count + + return len(tf.termFacets) +} +func (tf *TermFacets) Swap(i, j int) { + tf.termFacets[i], tf.termFacets[j] = tf.termFacets[j], tf.termFacets[i] +} +func (tf *TermFacets) Less(i, j int) bool { + if tf.termFacets[i].Count == tf.termFacets[j].Count { + return tf.termFacets[i].Term < tf.termFacets[j].Term + } + return tf.termFacets[i].Count > tf.termFacets[j].Count +} + +// TermFacets used to be a type alias for []*TermFacet. +// To maintain backwards compatibility, we have to implement custom +// JSON marshalling. +func (tf *TermFacets) MarshalJSON() ([]byte, error) { + return json.Marshal(tf.termFacets) +} + +func (tf *TermFacets) UnmarshalJSON(b []byte) error { + termFacets := []*TermFacet{} + err := json.Unmarshal(b, &termFacets) + if err != nil { + return err + } + + for _, termFacet := range termFacets { + tf.Add(termFacet) + } + + return nil } type NumericRangeFacet struct { @@ -246,7 +301,7 @@ type FacetResult struct { Total int `json:"total"` Missing int `json:"missing"` Other int `json:"other"` - Terms TermFacets `json:"terms,omitempty"` + Terms *TermFacets `json:"terms,omitempty"` NumericRanges NumericRangeFacets `json:"numeric_ranges,omitempty"` DateRanges DateRangeFacets `json:"date_ranges,omitempty"` } @@ -254,7 +309,7 @@ type FacetResult struct { func (fr *FacetResult) Size() int { return reflectStaticSizeFacetResult + size.SizeOfPtr + len(fr.Field) + - len(fr.Terms)*(reflectStaticSizeTermFacet+size.SizeOfPtr) + + fr.Terms.Len()*(reflectStaticSizeTermFacet+size.SizeOfPtr) + len(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) + len(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr) } @@ -264,8 +319,8 @@ func (fr *FacetResult) Merge(other *FacetResult) { fr.Missing += other.Missing fr.Other += other.Other if fr.Terms != nil && other.Terms != nil { - for _, term := range other.Terms { - fr.Terms = fr.Terms.Add(term) + for _, term := range other.Terms.termFacets { + fr.Terms.Add(term) } } if fr.NumericRanges != nil && other.NumericRanges != nil { @@ -283,12 +338,12 @@ func (fr *FacetResult) Merge(other *FacetResult) { func (fr *FacetResult) Fixup(size int) { if fr.Terms != nil { sort.Sort(fr.Terms) - if len(fr.Terms) > size { - moveToOther := fr.Terms[size:] + if fr.Terms.Len() > size { + moveToOther := fr.Terms.termFacets[size:] for _, mto := range moveToOther { fr.Other += mto.Count } - fr.Terms = fr.Terms[0:size] + fr.Terms.termFacets = fr.Terms.termFacets[0:size] } } else if fr.NumericRanges != nil { sort.Sort(fr.NumericRanges) diff --git a/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go index 348dc1ce34ed..34e5c9597eea 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go @@ -123,9 +123,15 @@ OUTER: // if there were no terms to highlight // produce a single fragment from the beginning start := 0 - end := start + s.fragmentSize - if end > len(orig) { - end = len(orig) + end := start + used := 0 + for end < len(orig) && used < s.fragmentSize { + r, size := utf8.DecodeRune(orig[end:]) + if r == utf8.RuneError { + break + } + end += size + used++ } rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end}) } diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go b/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go new file mode 100644 index 000000000000..b02b5ed2c30a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/ip_range.go @@ -0,0 +1,84 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "fmt" + "net" + + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/searcher" + index "github.com/blevesearch/bleve_index_api" +) + +type IPRangeQuery struct { + CIDR string `json:"cidr, omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewIPRangeQuery(cidr string) *IPRangeQuery { + return &IPRangeQuery{ + CIDR: cidr, + } +} + +func (q *IPRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *IPRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *IPRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *IPRangeQuery) Field() string { + return q.FieldVal +} + +func (q *IPRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + _, ipNet, err := net.ParseCIDR(q.CIDR) + if err != nil { + ip := net.ParseIP(q.CIDR) + if ip == nil { + return nil, err + } + // If we are searching for a specific ip rather than members of a network, just use a term search. + return searcher.NewTermSearcherBytes(i, ip.To16(), field, q.BoostVal.Value(), options) + } + return searcher.NewIPRangeSearcher(i, ipNet, field, q.BoostVal.Value(), options) +} + +func (q *IPRangeQuery) Validate() error { + _, _, err := net.ParseCIDR(q.CIDR) + if err == nil { + return nil + } + // We also allow search for a specific IP. + ip := net.ParseIP(q.CIDR) + if ip != nil { + return nil // we have a valid ip + } + return fmt.Errorf("IPRangeQuery must be for an network or ip address, %q", q.CIDR) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go index 3a9cf23983ca..c01fa6fc29cf 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/query/query_string_lex.go @@ -248,8 +248,8 @@ func inTildeState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { - // only a non-escaped space ends the tilde (or eof) - if eof || (!l.inEscape && next == ' ') { + // end on non-escaped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { // end number l.nextTokenType = tNUMBER l.nextToken = &yySymType{ @@ -257,7 +257,13 @@ func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } logDebugTokens("NUMBER - '%s'", l.nextToken.s) l.reset() - return startState, true + + consumed := true + if !eof && (next == ':' || next == '^' || next == '~') { + consumed = false + } + + return startState, consumed } else if !l.inEscape && next == '\\' { l.inEscape = true return inNumOrStrState, true @@ -287,7 +293,7 @@ func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { } func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { - // end on non-escped space, colon, tilde, boost (or eof) + // end on non-escaped space, colon, tilde, boost (or eof) if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { // end string l.nextTokenType = tSTRING diff --git a/vendor/github.com/blevesearch/bleve/v2/search/search.go b/vendor/github.com/blevesearch/bleve/v2/search/search.go index 9277d52959e8..d2dd33712e6c 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/search.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/search.go @@ -270,7 +270,7 @@ func (dm *DocumentMatch) Complete(prealloc []Location) []Location { var needsDedupe bool for i, ftl := range dm.FieldTermLocations { - if lastField != ftl.Field { + if i == 0 || lastField != ftl.Field { lastField = ftl.Field if dm.Locations == nil { diff --git a/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go new file mode 100644 index 000000000000..d62f058080d7 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go @@ -0,0 +1,67 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "net" + + "github.com/blevesearch/bleve/v2/search" + index "github.com/blevesearch/bleve_index_api" +) + +// netLimits returns the lo and hi bounds inside the network. +func netLimits(n *net.IPNet) (lo net.IP, hi net.IP) { + ones, bits := n.Mask.Size() + netNum := n.IP + if bits == net.IPv4len*8 { + netNum = netNum.To16() + ones += 8 * (net.IPv6len - net.IPv4len) + } + mask := net.CIDRMask(ones, 8*net.IPv6len) + lo = make(net.IP, net.IPv6len) + hi = make(net.IP, net.IPv6len) + for i := 0; i < net.IPv6len; i++ { + lo[i] = netNum[i] & mask[i] + hi[i] = lo[i] | ^mask[i] + } + return lo, hi +} + +func NewIPRangeSearcher(indexReader index.IndexReader, ipNet *net.IPNet, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + lo, hi := netLimits(ipNet) + fieldDict, err := indexReader.FieldDictRange(field, lo, hi) + if err != nil { + return nil, err + } + defer fieldDict.Close() + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr(field, len(terms)) + } + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) +} diff --git a/vendor/github.com/blevesearch/bleve/v2/search/sort.go b/vendor/github.com/blevesearch/bleve/v2/search/sort.go index 496db47fc023..9ed9a78c01c6 100644 --- a/vendor/github.com/blevesearch/bleve/v2/search/sort.go +++ b/vendor/github.com/blevesearch/bleve/v2/search/sort.go @@ -21,12 +21,13 @@ import ( "math" "sort" "strings" + "unicode/utf8" "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/numeric" ) -var HighTerm = strings.Repeat(string([]byte{0xff}), 10) +var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) var LowTerm = string([]byte{0x00}) type SearchSort interface { diff --git a/vendor/github.com/blevesearch/bleve_index_api/directory.go b/vendor/github.com/blevesearch/bleve_index_api/directory.go new file mode 100644 index 000000000000..709a38456547 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve_index_api/directory.go @@ -0,0 +1,23 @@ +// Copyright (c) 2021 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "io" +) + +type Directory interface { + GetWriter(filePath string) (io.WriteCloser, error) +} diff --git a/vendor/github.com/blevesearch/mmap-go/go.mod b/vendor/github.com/blevesearch/mmap-go/go.mod index b7f0265734d4..b976271cfa32 100644 --- a/vendor/github.com/blevesearch/mmap-go/go.mod +++ b/vendor/github.com/blevesearch/mmap-go/go.mod @@ -1,3 +1,5 @@ module github.com/blevesearch/mmap-go -require golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 +go 1.13 + +require golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a diff --git a/vendor/github.com/blevesearch/mmap-go/go.sum b/vendor/github.com/blevesearch/mmap-go/go.sum index db2bff5e0f8c..b1bd14d21de4 100644 --- a/vendor/github.com/blevesearch/mmap-go/go.sum +++ b/vendor/github.com/blevesearch/mmap-go/go.sum @@ -1,2 +1,2 @@ -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 h1:IcgEB62HYgAhX0Nd/QrVgZlxlcyxbGQHElLUhW2X4Fo= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod index fb60b845efc9..0546a589e8dd 100644 --- a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.mod @@ -3,8 +3,7 @@ module github.com/blevesearch/scorch_segment_api/v2 go 1.13 require ( - github.com/RoaringBitmap/roaring v0.4.23 - github.com/blevesearch/bleve_index_api v1.0.0 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 github.com/davecgh/go-spew v1.1.1 // indirect - github.com/mschoch/smat v0.2.0 // indirect ) diff --git a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum index c465e59d6972..df36323d38a7 100644 --- a/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum +++ b/vendor/github.com/blevesearch/scorch_segment_api/v2/go.sum @@ -1,34 +1,19 @@ -github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/vellum/go.mod b/vendor/github.com/blevesearch/vellum/go.mod index 4678aa8fa87a..a0797698b86b 100644 --- a/vendor/github.com/blevesearch/vellum/go.mod +++ b/vendor/github.com/blevesearch/vellum/go.mod @@ -1,10 +1,9 @@ module github.com/blevesearch/vellum -go 1.12 +go 1.13 require ( github.com/bits-and-blooms/bitset v1.2.0 - github.com/blevesearch/mmap-go v1.0.2 + github.com/blevesearch/mmap-go v1.0.3 github.com/spf13/cobra v0.0.5 - golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect ) diff --git a/vendor/github.com/blevesearch/vellum/go.sum b/vendor/github.com/blevesearch/vellum/go.sum index 676efb69c142..c14b99abd401 100644 --- a/vendor/github.com/blevesearch/vellum/go.sum +++ b/vendor/github.com/blevesearch/vellum/go.sum @@ -2,8 +2,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -32,7 +32,6 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/github.com/blevesearch/zapx/v11/build.go b/vendor/github.com/blevesearch/zapx/v11/build.go index a4b5d2117b07..3f13a2a6c6b2 100644 --- a/vendor/github.com/blevesearch/zapx/v11/build.go +++ b/vendor/github.com/blevesearch/zapx/v11/build.go @@ -16,6 +16,8 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,40 +58,58 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err } - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkFactor, sb.memCRC, br) + err = f.Sync() if err != nil { cleanup() return err } - err = br.Flush() + err = f.Close() if err != nil { cleanup() return err } - err = f.Sync() + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) if err != nil { - cleanup() - return err + return 0, err } - err = f.Close() + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkFactor, sb.memCRC, br) if err != nil { - cleanup() - return err + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err } - return nil + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zapx/v11/go.mod b/vendor/github.com/blevesearch/zapx/v11/go.mod index 74c989a62f5a..26f837ff85af 100644 --- a/vendor/github.com/blevesearch/zapx/v11/go.mod +++ b/vendor/github.com/blevesearch/zapx/v11/go.mod @@ -1,13 +1,13 @@ module github.com/blevesearch/zapx/v11 -go 1.12 +go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 - github.com/blevesearch/bleve_index_api v1.0.0 - github.com/blevesearch/mmap-go v1.0.2 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 - github.com/blevesearch/vellum v1.0.5 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 + github.com/blevesearch/vellum v1.0.7 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v0.0.5 ) diff --git a/vendor/github.com/blevesearch/zapx/v11/go.sum b/vendor/github.com/blevesearch/zapx/v11/go.sum index 1c0359feabf4..ab9b8f8b6d27 100644 --- a/vendor/github.com/blevesearch/zapx/v11/go.sum +++ b/vendor/github.com/blevesearch/zapx/v11/go.sum @@ -1,19 +1,17 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -21,29 +19,20 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -53,43 +42,17 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/zapx/v11/posting.go b/vendor/github.com/blevesearch/zapx/v11/posting.go index b7125e7c8ae1..6a8ead5264c0 100644 --- a/vendor/github.com/blevesearch/zapx/v11/posting.go +++ b/vendor/github.com/blevesearch/zapx/v11/posting.go @@ -657,13 +657,18 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, return 0, false, nil } - if i.postings == nil || i.postings.postings == i.ActualBM { + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { return i.nextDocNumAtOrAfterClean(atOrAfter) } i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - if !i.Actual.HasNext() { + if !i.Actual.HasNext() || !i.all.HasNext() { // couldn't find anything return 0, false, nil } @@ -688,6 +693,10 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, } } + if !i.all.HasNext() { + return 0, false, nil + } + allN = i.all.Next() } diff --git a/vendor/github.com/blevesearch/zapx/v12/build.go b/vendor/github.com/blevesearch/zapx/v12/build.go index eec4dde08c27..de8265c140ab 100644 --- a/vendor/github.com/blevesearch/zapx/v12/build.go +++ b/vendor/github.com/blevesearch/zapx/v12/build.go @@ -16,6 +16,8 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,40 +58,58 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err } - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) + err = f.Sync() if err != nil { cleanup() return err } - err = br.Flush() + err = f.Close() if err != nil { cleanup() return err } - err = f.Sync() + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) if err != nil { - cleanup() - return err + return 0, err } - err = f.Close() + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) if err != nil { - cleanup() - return err + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err } - return nil + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zapx/v12/go.mod b/vendor/github.com/blevesearch/zapx/v12/go.mod index 1ff20d9c8a8d..26fd15a08427 100644 --- a/vendor/github.com/blevesearch/zapx/v12/go.mod +++ b/vendor/github.com/blevesearch/zapx/v12/go.mod @@ -1,13 +1,13 @@ module github.com/blevesearch/zapx/v12 -go 1.12 +go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 - github.com/blevesearch/bleve_index_api v1.0.0 - github.com/blevesearch/mmap-go v1.0.2 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 - github.com/blevesearch/vellum v1.0.5 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 + github.com/blevesearch/vellum v1.0.7 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v0.0.5 ) diff --git a/vendor/github.com/blevesearch/zapx/v12/go.sum b/vendor/github.com/blevesearch/zapx/v12/go.sum index 1c0359feabf4..ab9b8f8b6d27 100644 --- a/vendor/github.com/blevesearch/zapx/v12/go.sum +++ b/vendor/github.com/blevesearch/zapx/v12/go.sum @@ -1,19 +1,17 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -21,29 +19,20 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -53,43 +42,17 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/zapx/v12/posting.go b/vendor/github.com/blevesearch/zapx/v12/posting.go index d6c61a42ce1f..abae0cc3105f 100644 --- a/vendor/github.com/blevesearch/zapx/v12/posting.go +++ b/vendor/github.com/blevesearch/zapx/v12/posting.go @@ -536,13 +536,18 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, return 0, false, nil } - if i.postings == nil || i.postings.postings == i.ActualBM { + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { return i.nextDocNumAtOrAfterClean(atOrAfter) } i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - if !i.Actual.HasNext() { + if !i.Actual.HasNext() || !i.all.HasNext() { // couldn't find anything return 0, false, nil } @@ -571,6 +576,10 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, } } + if !i.all.HasNext() { + return 0, false, nil + } + allN = i.all.Next() } diff --git a/vendor/github.com/blevesearch/zapx/v13/build.go b/vendor/github.com/blevesearch/zapx/v13/build.go index 5a25eef77bbd..827e5c47e87c 100644 --- a/vendor/github.com/blevesearch/zapx/v13/build.go +++ b/vendor/github.com/blevesearch/zapx/v13/build.go @@ -16,6 +16,8 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,40 +58,58 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err } - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) + err = f.Sync() if err != nil { cleanup() return err } - err = br.Flush() + err = f.Close() if err != nil { cleanup() return err } - err = f.Sync() + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) if err != nil { - cleanup() - return err + return 0, err } - err = f.Close() + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) if err != nil { - cleanup() - return err + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err } - return nil + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zapx/v13/go.mod b/vendor/github.com/blevesearch/zapx/v13/go.mod index 5e947d25ecb7..267e75ca782f 100644 --- a/vendor/github.com/blevesearch/zapx/v13/go.mod +++ b/vendor/github.com/blevesearch/zapx/v13/go.mod @@ -1,13 +1,13 @@ module github.com/blevesearch/zapx/v13 -go 1.12 +go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 - github.com/blevesearch/bleve_index_api v1.0.0 - github.com/blevesearch/mmap-go v1.0.2 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 - github.com/blevesearch/vellum v1.0.5 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 + github.com/blevesearch/vellum v1.0.7 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v0.0.5 ) diff --git a/vendor/github.com/blevesearch/zapx/v13/go.sum b/vendor/github.com/blevesearch/zapx/v13/go.sum index 1c0359feabf4..ab9b8f8b6d27 100644 --- a/vendor/github.com/blevesearch/zapx/v13/go.sum +++ b/vendor/github.com/blevesearch/zapx/v13/go.sum @@ -1,19 +1,17 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -21,29 +19,20 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -53,43 +42,17 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/zapx/v13/posting.go b/vendor/github.com/blevesearch/zapx/v13/posting.go index d6c61a42ce1f..abae0cc3105f 100644 --- a/vendor/github.com/blevesearch/zapx/v13/posting.go +++ b/vendor/github.com/blevesearch/zapx/v13/posting.go @@ -536,13 +536,18 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, return 0, false, nil } - if i.postings == nil || i.postings.postings == i.ActualBM { + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { return i.nextDocNumAtOrAfterClean(atOrAfter) } i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - if !i.Actual.HasNext() { + if !i.Actual.HasNext() || !i.all.HasNext() { // couldn't find anything return 0, false, nil } @@ -571,6 +576,10 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, } } + if !i.all.HasNext() { + return 0, false, nil + } + allN = i.all.Next() } diff --git a/vendor/github.com/blevesearch/zapx/v14/build.go b/vendor/github.com/blevesearch/zapx/v14/build.go index 59c426848871..b36878abbb91 100644 --- a/vendor/github.com/blevesearch/zapx/v14/build.go +++ b/vendor/github.com/blevesearch/zapx/v14/build.go @@ -16,6 +16,8 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,40 +58,58 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err } - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) + err = f.Sync() if err != nil { cleanup() return err } - err = br.Flush() + err = f.Close() if err != nil { cleanup() return err } - err = f.Sync() + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) if err != nil { - cleanup() - return err + return 0, err } - err = f.Close() + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) if err != nil { - cleanup() - return err + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err } - return nil + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zapx/v14/go.mod b/vendor/github.com/blevesearch/zapx/v14/go.mod index ee4f7d0a362f..c3e526893fd0 100644 --- a/vendor/github.com/blevesearch/zapx/v14/go.mod +++ b/vendor/github.com/blevesearch/zapx/v14/go.mod @@ -1,13 +1,13 @@ module github.com/blevesearch/zapx/v14 -go 1.12 +go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 - github.com/blevesearch/bleve_index_api v1.0.0 - github.com/blevesearch/mmap-go v1.0.2 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 - github.com/blevesearch/vellum v1.0.5 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 + github.com/blevesearch/vellum v1.0.7 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v0.0.5 ) diff --git a/vendor/github.com/blevesearch/zapx/v14/go.sum b/vendor/github.com/blevesearch/zapx/v14/go.sum index 1c0359feabf4..ab9b8f8b6d27 100644 --- a/vendor/github.com/blevesearch/zapx/v14/go.sum +++ b/vendor/github.com/blevesearch/zapx/v14/go.sum @@ -1,19 +1,17 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -21,29 +19,20 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -53,43 +42,17 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/zapx/v14/posting.go b/vendor/github.com/blevesearch/zapx/v14/posting.go index 1d04247fb691..28e8b64c1147 100644 --- a/vendor/github.com/blevesearch/zapx/v14/posting.go +++ b/vendor/github.com/blevesearch/zapx/v14/posting.go @@ -544,13 +544,18 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, return 0, false, nil } - if i.postings == nil || i.postings.postings == i.ActualBM { + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { return i.nextDocNumAtOrAfterClean(atOrAfter) } i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - if !i.Actual.HasNext() { + if !i.Actual.HasNext() || !i.all.HasNext() { // couldn't find anything return 0, false, nil } @@ -574,6 +579,10 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, } } + if !i.all.HasNext() { + return 0, false, nil + } + allN = i.all.Next() } diff --git a/vendor/github.com/blevesearch/zapx/v15/build.go b/vendor/github.com/blevesearch/zapx/v15/build.go index 14309f90ca90..5db1d9ee2420 100644 --- a/vendor/github.com/blevesearch/zapx/v15/build.go +++ b/vendor/github.com/blevesearch/zapx/v15/build.go @@ -16,6 +16,8 @@ package zap import ( "bufio" + "fmt" + "io" "math" "os" @@ -32,6 +34,16 @@ func (sb *SegmentBase) Persist(path string) error { return PersistSegmentBase(sb, path) } +// WriteTo is an implementation of io.WriterTo interface. +func (sb *SegmentBase) WriteTo(w io.Writer) (int64, error) { + if w == nil { + return 0, fmt.Errorf("invalid writer found") + } + + n, err := persistSegmentBaseToWriter(sb, w) + return int64(n), err +} + // PersistSegmentBase persists SegmentBase in the zap file format. func PersistSegmentBase(sb *SegmentBase, path string) error { flag := os.O_RDWR | os.O_CREATE @@ -46,40 +58,58 @@ func PersistSegmentBase(sb *SegmentBase, path string) error { _ = os.Remove(path) } - br := bufio.NewWriter(f) - - _, err = br.Write(sb.mem) + _, err = persistSegmentBaseToWriter(sb, f) if err != nil { cleanup() return err } - err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, - sb.chunkMode, sb.memCRC, br) + err = f.Sync() if err != nil { cleanup() return err } - err = br.Flush() + err = f.Close() if err != nil { cleanup() return err } - err = f.Sync() + return err +} + +type bufWriter struct { + w *bufio.Writer + n int +} + +func (br *bufWriter) Write(in []byte) (int, error) { + n, err := br.w.Write(in) + br.n += n + return n, err +} + +func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) { + br := &bufWriter{w: bufio.NewWriter(w)} + + _, err := br.Write(sb.mem) if err != nil { - cleanup() - return err + return 0, err } - err = f.Close() + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, + sb.docValueOffset, sb.chunkMode, sb.memCRC, br) if err != nil { - cleanup() - return err + return 0, err + } + + err = br.w.Flush() + if err != nil { + return 0, err } - return nil + return br.n, nil } func persistStoredFieldValues(fieldID int, diff --git a/vendor/github.com/blevesearch/zapx/v15/go.mod b/vendor/github.com/blevesearch/zapx/v15/go.mod index e74519d6d5f6..fd31ab5910bb 100644 --- a/vendor/github.com/blevesearch/zapx/v15/go.mod +++ b/vendor/github.com/blevesearch/zapx/v15/go.mod @@ -1,13 +1,14 @@ module github.com/blevesearch/zapx/v15 -go 1.12 +go 1.13 require ( - github.com/RoaringBitmap/roaring v0.7.3 - github.com/blevesearch/bleve_index_api v1.0.0 - github.com/blevesearch/mmap-go v1.0.2 - github.com/blevesearch/scorch_segment_api/v2 v2.0.1 - github.com/blevesearch/vellum v1.0.5 + github.com/RoaringBitmap/roaring v0.9.4 + github.com/blevesearch/bleve_index_api v1.0.1 + github.com/blevesearch/mmap-go v1.0.3 + github.com/blevesearch/scorch_segment_api/v2 v2.1.0 + github.com/blevesearch/vellum v1.0.7 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v0.0.5 + golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect ) diff --git a/vendor/github.com/blevesearch/zapx/v15/go.sum b/vendor/github.com/blevesearch/zapx/v15/go.sum index 1c0359feabf4..8ce24a355b5f 100644 --- a/vendor/github.com/blevesearch/zapx/v15/go.sum +++ b/vendor/github.com/blevesearch/zapx/v15/go.sum @@ -1,19 +1,17 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/RoaringBitmap/roaring v0.7.3 h1:RwirWpvFONt2EwHHEHhER7S4BHZkyj3qL5LXLlnQPZ4= -github.com/RoaringBitmap/roaring v0.7.3/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= +github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= +github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/bleve_index_api v1.0.0 h1:Ds3XeuTxjXCkG6pgIwWDRyooJKNIuOKemnN0N0IkhTU= -github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1 h1:fd+hPtZ8GsbqPK1HslGp7Vhoik4arZteA/IsCEgOisw= -github.com/blevesearch/scorch_segment_api/v2 v2.0.1/go.mod h1:lq7yK2jQy1yQjtjTfU931aVqz7pYxEudHaDwOt1tXfU= -github.com/blevesearch/vellum v1.0.5 h1:L5dJ7hKauRVbuH7I8uqLeSK92CPPY6FfrbAmLhAug8A= -github.com/blevesearch/vellum v1.0.5/go.mod h1:atE0EH3fvk43zzS7t1YNdNC7DbmcC3uz+eMD5xZ2OyQ= +github.com/blevesearch/bleve_index_api v1.0.1 h1:nx9++0hnyiGOHJwQQYfsUGzpRdEVE5LsylmmngQvaFk= +github.com/blevesearch/bleve_index_api v1.0.1/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= +github.com/blevesearch/mmap-go v1.0.3 h1:7QkALgFNooSq3a46AE+pWeKASAZc9SiNFJhDGF1NDx4= +github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0 h1:NFwteOpZEvJk5Vg0H6gD0hxupsG3JYocE4DBvsA2GZI= +github.com/blevesearch/scorch_segment_api/v2 v2.1.0/go.mod h1:uch7xyyO/Alxkuxa+CGs79vw0QY8BENSBjg6Mw5L5DE= +github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= +github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -21,29 +19,20 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -53,43 +42,18 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/vendor/github.com/blevesearch/zapx/v15/posting.go b/vendor/github.com/blevesearch/zapx/v15/posting.go index b1d19e532d3c..484bd0e148ef 100644 --- a/vendor/github.com/blevesearch/zapx/v15/posting.go +++ b/vendor/github.com/blevesearch/zapx/v15/posting.go @@ -562,13 +562,18 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, return 0, false, nil } - if i.postings == nil || i.postings.postings == i.ActualBM { + if i.postings == nil || i.postings == emptyPostingsList { + // couldn't find anything + return 0, false, nil + } + + if i.postings.postings == i.ActualBM { return i.nextDocNumAtOrAfterClean(atOrAfter) } i.Actual.AdvanceIfNeeded(uint32(atOrAfter)) - if !i.Actual.HasNext() { + if !i.Actual.HasNext() || !i.all.HasNext() { // couldn't find anything return 0, false, nil } @@ -592,6 +597,10 @@ func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, } } + if !i.all.HasNext() { + return 0, false, nil + } + allN = i.all.Next() } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8b2b9350b7f6..dfc355e9040a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -66,8 +66,7 @@ github.com/PuerkitoBio/goquery github.com/PuerkitoBio/purell # github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 github.com/PuerkitoBio/urlesc -# github.com/RoaringBitmap/roaring v0.9.1 -## explicit +# github.com/RoaringBitmap/roaring v0.9.4 github.com/RoaringBitmap/roaring github.com/RoaringBitmap/roaring/internal # github.com/acomagu/bufpipe v1.0.3 @@ -122,7 +121,7 @@ github.com/aymerick/douceur/parser github.com/beorn7/perks/quantile # github.com/bits-and-blooms/bitset v1.2.0 github.com/bits-and-blooms/bitset -# github.com/blevesearch/bleve/v2 v2.0.6 +# github.com/blevesearch/bleve/v2 v2.3.0 ## explicit github.com/blevesearch/bleve/v2 github.com/blevesearch/bleve/v2/analysis @@ -160,13 +159,13 @@ github.com/blevesearch/bleve/v2/search/query github.com/blevesearch/bleve/v2/search/scorer github.com/blevesearch/bleve/v2/search/searcher github.com/blevesearch/bleve/v2/size -# github.com/blevesearch/bleve_index_api v1.0.0 +# github.com/blevesearch/bleve_index_api v1.0.1 github.com/blevesearch/bleve_index_api # github.com/blevesearch/go-porterstemmer v1.0.3 github.com/blevesearch/go-porterstemmer -# github.com/blevesearch/mmap-go v1.0.2 +# github.com/blevesearch/mmap-go v1.0.3 github.com/blevesearch/mmap-go -# github.com/blevesearch/scorch_segment_api/v2 v2.0.1 +# github.com/blevesearch/scorch_segment_api/v2 v2.1.0 github.com/blevesearch/scorch_segment_api/v2 # github.com/blevesearch/segment v0.9.0 github.com/blevesearch/segment @@ -175,20 +174,20 @@ github.com/blevesearch/snowballstem github.com/blevesearch/snowballstem/english # github.com/blevesearch/upsidedown_store_api v1.0.1 github.com/blevesearch/upsidedown_store_api -# github.com/blevesearch/vellum v1.0.5 +# github.com/blevesearch/vellum v1.0.7 github.com/blevesearch/vellum github.com/blevesearch/vellum/levenshtein github.com/blevesearch/vellum/regexp github.com/blevesearch/vellum/utf8 -# github.com/blevesearch/zapx/v11 v11.2.1 +# github.com/blevesearch/zapx/v11 v11.3.2 github.com/blevesearch/zapx/v11 -# github.com/blevesearch/zapx/v12 v12.2.1 +# github.com/blevesearch/zapx/v12 v12.3.2 github.com/blevesearch/zapx/v12 -# github.com/blevesearch/zapx/v13 v13.2.1 +# github.com/blevesearch/zapx/v13 v13.3.2 github.com/blevesearch/zapx/v13 -# github.com/blevesearch/zapx/v14 v14.2.1 +# github.com/blevesearch/zapx/v14 v14.3.2 github.com/blevesearch/zapx/v14 -# github.com/blevesearch/zapx/v15 v15.2.1 +# github.com/blevesearch/zapx/v15 v15.3.2 github.com/blevesearch/zapx/v15 # github.com/boombuler/barcode v1.0.1 ## explicit From 385dc6a9927bd4bb66cb2a62e3f7e5643b973ee9 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 1 Jan 2022 17:05:31 +0800 Subject: [PATCH 02/22] Allow admin to associate missing LFS objects for repositories (#18143) This PR reworked the Find pointer files feature in Settings -> LFS page. When a LFS object is missing from database but exists in LFS content store, admin can associate it to the repository by clicking the Associate button. This PR is not perfect (because the LFS module itself should be improved too), it's just a nice-to-have feature to help users recover their LFS repositories (eg: database was lost / table was truncated) --- models/lfs.go | 53 ++++++++++++++++------- routers/web/repo/lfs.go | 28 +++++++----- templates/repo/settings/lfs_pointers.tmpl | 2 +- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 56924ffcf2d5..cf596f546838 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -7,11 +7,13 @@ package models import ( "context" "errors" + "fmt" "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -145,6 +147,11 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) { return count > 0, err } +// LFSObjectIsAssociated checks if a provided Oid is associated +func LFSObjectIsAssociated(oid string) (bool, error) { + return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) +} + // LFSAutoAssociate auto associates accessible LFSMetaObjects func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error { ctx, committer, err := db.TxContext() @@ -162,23 +169,39 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6 oidMap[meta.Oid] = meta } - cond := builder.NewCond() if !user.IsAdmin { - cond = builder.In("`lfs_meta_object`.repository_id", - builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user))) - } - newMetas := make([]*LFSMetaObject, 0, len(metas)) - if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil { - return err - } - for i := range newMetas { - newMetas[i].Size = oidMap[newMetas[i].Oid].Size - newMetas[i].RepositoryID = repoID - } - if err := db.Insert(ctx, newMetas); err != nil { - return err + newMetas := make([]*LFSMetaObject, 0, len(metas)) + cond := builder.In( + "`lfs_meta_object`.repository_id", + builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)), + ) + err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas) + if err != nil { + return err + } + if len(newMetas) != len(oidMap) { + return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas)) + } + for i := range newMetas { + newMetas[i].Size = oidMap[newMetas[i].Oid].Size + newMetas[i].RepositoryID = repoID + } + if err = db.Insert(ctx, newMetas); err != nil { + return err + } + } else { + // admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key), + // even if error occurs, it won't hurt users and won't make things worse + for i := range metas { + _, err = sess.Insert(&LFSMetaObject{ + Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size}, + RepositoryID: repoID, + }) + if err != nil { + log.Warn("failed to insert LFS meta object into database, err=%v", err) + } + } } - return committer.Commit() } diff --git a/routers/web/repo/lfs.go b/routers/web/repo/lfs.go index 28d6b12860af..6cc05430dde0 100644 --- a/routers/web/repo/lfs.go +++ b/routers/web/repo/lfs.go @@ -421,12 +421,13 @@ func LFSPointerFiles(ctx *context.Context) { var numAssociated, numNoExist, numAssociatable int type pointerResult struct { - SHA string - Oid string - Size int64 - InRepo bool - Exists bool - Accessible bool + SHA string + Oid string + Size int64 + InRepo bool + Exists bool + Accessible bool + Associatable bool } results := []pointerResult{} @@ -461,22 +462,29 @@ func LFSPointerFiles(ctx *context.Context) { // Can we fix? // OK well that's "simple" // - we need to check whether current user has access to a repo that has access to the file - result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) + result.Associatable, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) if err != nil { return err } - } else { - result.Accessible = true + if !result.Associatable { + associated, err := models.LFSObjectIsAssociated(pointerBlob.Oid) + if err != nil { + return err + } + result.Associatable = !associated + } } } + result.Accessible = result.InRepo || result.Associatable + if result.InRepo { numAssociated++ } if !result.Exists { numNoExist++ } - if !result.InRepo && result.Accessible { + if result.Associatable { numAssociatable++ } diff --git a/templates/repo/settings/lfs_pointers.tmpl b/templates/repo/settings/lfs_pointers.tmpl index bf23062ae3cc..440e54423255 100644 --- a/templates/repo/settings/lfs_pointers.tmpl +++ b/templates/repo/settings/lfs_pointers.tmpl @@ -11,7 +11,7 @@
{{.CsrfTokenHtml}} {{range .Pointers}} - {{if and (not .InRepo) .Exists .Accessible}} + {{if .Associatable}} {{end}} {{end}} From e9c9a35a61385894754dd22bcf4e1bed4efe7744 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 1 Jan 2022 17:43:28 +0800 Subject: [PATCH 03/22] Upgrade certmagic from v0.14.1 to v0.15.2 (#18138) --- cmd/web_letsencrypt.go | 3 +- go.mod | 3 +- go.sum | 10 +- .../caddyserver/certmagic/README.md | 16 +-- .../caddyserver/certmagic/account.go | 4 - .../caddyserver/certmagic/acmeclient.go | 4 +- .../github.com/caddyserver/certmagic/cache.go | 44 ++++++- .../caddyserver/certmagic/certificates.go | 2 - .../caddyserver/certmagic/certmagic.go | 10 +- .../caddyserver/certmagic/config.go | 76 ++++++----- .../caddyserver/certmagic/crypto.go | 12 +- .../github.com/caddyserver/certmagic/go.mod | 6 +- .../github.com/caddyserver/certmagic/go.sum | 12 +- .../caddyserver/certmagic/handshake.go | 120 ++++++++++++++---- .../caddyserver/certmagic/solvers.go | 39 +++--- .../mholt/acmez/acme/certificate.go | 2 +- vendor/github.com/mholt/acmez/acme/http.go | 7 +- vendor/github.com/mholt/acmez/acme/problem.go | 40 +++++- vendor/github.com/mholt/acmez/client.go | 21 +-- vendor/modules.txt | 5 +- 20 files changed, 294 insertions(+), 142 deletions(-) diff --git a/cmd/web_letsencrypt.go b/cmd/web_letsencrypt.go index 066208457bae..866b88f7c9d5 100644 --- a/cmd/web_letsencrypt.go +++ b/cmd/web_letsencrypt.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -47,7 +48,7 @@ func runLetsEncrypt(listenAddr, domain, directory, email string, m http.Handler) magic.Issuers = []certmagic.Issuer{myACME} // this obtains certificates or renews them if necessary - err := magic.ManageSync([]string{domain}) + err := magic.ManageSync(graceful.GetManager().HammerContext(), []string{domain}) if err != nil { return err } diff --git a/go.mod b/go.mod index 70eb74e99767..070ab653251f 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.0 github.com/boombuler/barcode v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect - github.com/caddyserver/certmagic v0.14.1 + github.com/caddyserver/certmagic v0.15.2 github.com/chi-middleware/proxy v1.1.1 github.com/couchbase/go-couchbase v0.0.0-20210224140812-5740cd35f448 // indirect github.com/couchbase/gomemcached v0.1.2 // indirect @@ -78,7 +78,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.8 github.com/mholt/archiver/v3 v3.5.0 github.com/microcosm-cc/bluemonday v1.0.16 - github.com/miekg/dns v1.1.43 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.12 github.com/minio/sha256-simd v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7296a6574e33..c9c37e70c0ae 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= -github.com/caddyserver/certmagic v0.14.1 h1:8RIFS/LbGne/I7Op56Kkm2annnei7io9VW/IWDttE9U= -github.com/caddyserver/certmagic v0.14.1/go.mod h1:oRQOZmUVKwlpgNidslysHt05osM9uMrJ4YMk+Ot4P4Q= +github.com/caddyserver/certmagic v0.15.2 h1:OMTakTsLM1ZfzMDjwvYprfUgFzpVPh3u87oxMPwmeBc= +github.com/caddyserver/certmagic v0.15.2/go.mod h1:qhkAOthf72ufAcp3Y5jF2RaGE96oip3UbEQRIzwe3/8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -767,7 +767,6 @@ github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -848,14 +847,13 @@ github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxz github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mholt/acmez v0.1.3 h1:J7MmNIk4Qf9b8mAGqAh4XkNeowv3f1zW816yf4zt7Qk= -github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= +github.com/mholt/acmez v1.0.1 h1:J7uquHOKEmo71UDnVApy1sSLA0oF/r+NtVrNzMKKA9I= +github.com/mholt/acmez v1.0.1/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc= github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= diff --git a/vendor/github.com/caddyserver/certmagic/README.md b/vendor/github.com/caddyserver/certmagic/README.md index 19aa9f891963..9948d5483037 100644 --- a/vendor/github.com/caddyserver/certmagic/README.md +++ b/vendor/github.com/caddyserver/certmagic/README.md @@ -10,9 +10,9 @@

-Caddy's automagic TLS features—now for your own Go programs—in one powerful and easy-to-use library! +Caddy's [automagic TLS features](https://caddyserver.com/docs/automatic-https)—now for your own Go programs—in one powerful and easy-to-use library! -CertMagic is the most mature, robust, and capable ACME client integration for Go... and perhaps ever. +CertMagic is the most mature, robust, and powerful ACME client integration for Go... and perhaps ever. With CertMagic, you can add one line to your Go application to serve securely over TLS, without ever having to touch certificates. @@ -40,11 +40,6 @@ Compared to other ACME client libraries for Go, only CertMagic supports the full CertMagic - Automatic HTTPS using Let's Encrypt =============================================== -**Sponsored by Relica - Cross-platform local and cloud file backup:** - -Relica - Cross-platform file backup to the cloud, local disks, or other computers - - ## Menu - [Features](#features) @@ -116,6 +111,7 @@ CertMagic - Automatic HTTPS using Let's Encrypt ## Requirements +0. ACME server (can be a publicly-trusted CA, or your own) 1. Public DNS name(s) you control 2. Server reachable from public Internet - Or use the DNS challenge to waive this requirement @@ -270,7 +266,7 @@ myACME := certmagic.NewACMEManager(magic, certmagic.ACMEManager{ magic.Issuer = myACME // this obtains certificates or renews them if necessary -err := magic.ManageSync([]string{"example.com", "sub.example.com"}) +err := magic.ManageSync(context.TODO(), []string{"example.com", "sub.example.com"}) if err != nil { return err } @@ -279,6 +275,10 @@ if err != nil { // you can get a TLS config to use in a TLS listener! tlsConfig := magic.TLSConfig() +// be sure to customize NextProtos if serving a specific +// application protocol after the TLS handshake, for example: +tlsConfig.NextProtos = append([]string{"h2", "http/1.1"}, tlsConfig.NextProtos...) + //// OR //// // if you already have a TLS config you don't want to replace, diff --git a/vendor/github.com/caddyserver/certmagic/account.go b/vendor/github.com/caddyserver/certmagic/account.go index 8633f92fcbe9..1cc6723429a2 100644 --- a/vendor/github.com/caddyserver/certmagic/account.go +++ b/vendor/github.com/caddyserver/certmagic/account.go @@ -405,10 +405,6 @@ var ( discoveredEmailMu sync.Mutex ) -// agreementTestURL is set during tests to skip requiring -// setting up an entire ACME CA endpoint. -var agreementTestURL string - // stdin is used to read the user's input if prompted; // this is changed by tests during tests. var stdin = io.ReadWriter(os.Stdin) diff --git a/vendor/github.com/caddyserver/certmagic/acmeclient.go b/vendor/github.com/caddyserver/certmagic/acmeclient.go index cc876b9252b0..a22dc19a7ae0 100644 --- a/vendor/github.com/caddyserver/certmagic/acmeclient.go +++ b/vendor/github.com/caddyserver/certmagic/acmeclient.go @@ -370,11 +370,11 @@ var ( // RateLimitEvents is how many new events can be allowed // in RateLimitEventsWindow. - RateLimitEvents = 20 + RateLimitEvents = 10 // RateLimitEventsWindow is the size of the sliding // window that throttles events. - RateLimitEventsWindow = 1 * time.Minute + RateLimitEventsWindow = 10 * time.Second ) // Some default values passed down to the underlying ACME client. diff --git a/vendor/github.com/caddyserver/certmagic/cache.go b/vendor/github.com/caddyserver/certmagic/cache.go index 30bfc2a3f7d0..4e986303a9bd 100644 --- a/vendor/github.com/caddyserver/certmagic/cache.go +++ b/vendor/github.com/caddyserver/certmagic/cache.go @@ -194,6 +194,14 @@ func (certCache *Cache) cacheCertificate(cert Certificate) { func (certCache *Cache) unsyncedCacheCertificate(cert Certificate) { // no-op if this certificate already exists in the cache if _, ok := certCache.cache[cert.hash]; ok { + if certCache.logger != nil { + certCache.logger.Debug("certificate already cached", + zap.Strings("subjects", cert.Names), + zap.Time("expiration", cert.Leaf.NotAfter), + zap.Bool("managed", cert.managed), + zap.String("issuer_key", cert.issuerKey), + zap.String("hash", cert.hash)) + } return } @@ -209,6 +217,13 @@ func (certCache *Cache) unsyncedCacheCertificate(cert Certificate) { i := 0 for _, randomCert := range certCache.cache { if i == rnd { + if certCache.logger != nil { + certCache.logger.Debug("cache full; evicting random certificate", + zap.Strings("removing_subjects", randomCert.Names), + zap.String("removing_hash", randomCert.hash), + zap.Strings("inserting_subjects", cert.Names), + zap.String("inserting_hash", cert.hash)) + } certCache.removeCertificate(randomCert) break } @@ -223,6 +238,17 @@ func (certCache *Cache) unsyncedCacheCertificate(cert Certificate) { for _, name := range cert.Names { certCache.cacheIndex[name] = append(certCache.cacheIndex[name], cert.hash) } + + if certCache.logger != nil { + certCache.logger.Debug("added certificate to cache", + zap.Strings("subjects", cert.Names), + zap.Time("expiration", cert.Leaf.NotAfter), + zap.Bool("managed", cert.managed), + zap.String("issuer_key", cert.issuerKey), + zap.String("hash", cert.hash), + zap.Int("cache_size", len(certCache.cache)), + zap.Int("cache_capacity", certCache.options.Capacity)) + } } // removeCertificate removes cert from the cache. @@ -233,9 +259,10 @@ func (certCache *Cache) removeCertificate(cert Certificate) { // delete all mentions of this cert from the name index for _, name := range cert.Names { keyList := certCache.cacheIndex[name] - for i, cacheKey := range keyList { - if cacheKey == cert.hash { + for i := 0; i < len(keyList); i++ { + if keyList[i] == cert.hash { keyList = append(keyList[:i], keyList[i+1:]...) + i-- } } if len(keyList) == 0 { @@ -247,6 +274,17 @@ func (certCache *Cache) removeCertificate(cert Certificate) { // delete the actual cert from the cache delete(certCache.cache, cert.hash) + + if certCache.logger != nil { + certCache.logger.Debug("removed certificate from cache", + zap.Strings("subjects", cert.Names), + zap.Time("expiration", cert.Leaf.NotAfter), + zap.Bool("managed", cert.managed), + zap.String("issuer_key", cert.issuerKey), + zap.String("hash", cert.hash), + zap.Int("cache_size", len(certCache.cache)), + zap.Int("cache_capacity", certCache.options.Capacity)) + } } // replaceCertificate atomically replaces oldCert with newCert in @@ -260,7 +298,7 @@ func (certCache *Cache) replaceCertificate(oldCert, newCert Certificate) { certCache.mu.Unlock() if certCache.logger != nil { certCache.logger.Info("replaced certificate in cache", - zap.Strings("identifiers", newCert.Names), + zap.Strings("subjects", newCert.Names), zap.Time("new_expiration", newCert.Leaf.NotAfter)) } } diff --git a/vendor/github.com/caddyserver/certmagic/certificates.go b/vendor/github.com/caddyserver/certmagic/certificates.go index e6dab5e0ff82..067bfc508bd3 100644 --- a/vendor/github.com/caddyserver/certmagic/certificates.go +++ b/vendor/github.com/caddyserver/certmagic/certificates.go @@ -283,8 +283,6 @@ func fillCertFromLeaf(cert *Certificate, tlsCert tls.Certificate) error { return fmt.Errorf("certificate has no names") } - // save the hash of this certificate (chain) and - // expiration date, for necessity and efficiency cert.hash = hashCertificateChain(cert.Certificate.Certificate) return nil diff --git a/vendor/github.com/caddyserver/certmagic/certmagic.go b/vendor/github.com/caddyserver/certmagic/certmagic.go index bb33b90c559d..88080a8036df 100644 --- a/vendor/github.com/caddyserver/certmagic/certmagic.go +++ b/vendor/github.com/caddyserver/certmagic/certmagic.go @@ -73,7 +73,7 @@ func HTTPS(domainNames []string, mux http.Handler) error { DefaultACME.Agreed = true cfg := NewDefault() - err := cfg.ManageSync(domainNames) + err := cfg.ManageSync(context.Background(), domainNames) if err != nil { return err } @@ -178,7 +178,7 @@ func TLS(domainNames []string) (*tls.Config, error) { DefaultACME.Agreed = true DefaultACME.DisableHTTPChallenge = true cfg := NewDefault() - return cfg.TLSConfig(), cfg.ManageSync(domainNames) + return cfg.TLSConfig(), cfg.ManageSync(context.Background(), domainNames) } // Listen manages certificates for domainName and returns a @@ -195,7 +195,7 @@ func Listen(domainNames []string) (net.Listener, error) { DefaultACME.Agreed = true DefaultACME.DisableHTTPChallenge = true cfg := NewDefault() - err := cfg.ManageSync(domainNames) + err := cfg.ManageSync(context.Background(), domainNames) if err != nil { return nil, err } @@ -223,9 +223,9 @@ func Listen(domainNames []string) (net.Listener, error) { // // Calling this function signifies your acceptance to // the CA's Subscriber Agreement and/or Terms of Service. -func ManageSync(domainNames []string) error { +func ManageSync(ctx context.Context, domainNames []string) error { DefaultACME.Agreed = true - return NewDefault().ManageSync(domainNames) + return NewDefault().ManageSync(ctx, domainNames) } // ManageAsync is the same as ManageSync, except that diff --git a/vendor/github.com/caddyserver/certmagic/config.go b/vendor/github.com/caddyserver/certmagic/config.go index d408418f6571..3d031fa627ec 100644 --- a/vendor/github.com/caddyserver/certmagic/config.go +++ b/vendor/github.com/caddyserver/certmagic/config.go @@ -247,8 +247,28 @@ func newWithCache(certCache *Cache, cfg Config) *Config { // of the given domainNames. This behavior is recommended for // interactive use (i.e. when an administrator is present) so // that errors can be reported and fixed immediately. -func (cfg *Config) ManageSync(domainNames []string) error { - return cfg.manageAll(context.Background(), domainNames, false) +func (cfg *Config) ManageSync(ctx context.Context, domainNames []string) error { + return cfg.manageAll(ctx, domainNames, false) +} + +// ManageAsync is the same as ManageSync, except that ACME +// operations are performed asynchronously (in the background). +// This method returns before certificates are ready. It is +// crucial that the administrator monitors the logs and is +// notified of any errors so that corrective action can be +// taken as soon as possible. Any errors returned from this +// method occurred before ACME transactions started. +// +// As long as logs are monitored, this method is typically +// recommended for non-interactive environments. +// +// If there are failures loading, obtaining, or renewing a +// certificate, it will be retried with exponential backoff +// for up to about 30 days, with a maximum interval of about +// 24 hours. Cancelling ctx will cancel retries and shut down +// any goroutines spawned by ManageAsync. +func (cfg *Config) ManageAsync(ctx context.Context, domainNames []string) error { + return cfg.manageAll(ctx, domainNames, true) } // ClientCredentials returns a list of TLS client certificate chains for the given identifiers. @@ -274,26 +294,6 @@ func (cfg *Config) ClientCredentials(ctx context.Context, identifiers []string) return chains, nil } -// ManageAsync is the same as ManageSync, except that ACME -// operations are performed asynchronously (in the background). -// This method returns before certificates are ready. It is -// crucial that the administrator monitors the logs and is -// notified of any errors so that corrective action can be -// taken as soon as possible. Any errors returned from this -// method occurred before ACME transactions started. -// -// As long as logs are monitored, this method is typically -// recommended for non-interactive environments. -// -// If there are failures loading, obtaining, or renewing a -// certificate, it will be retried with exponential backoff -// for up to about 30 days, with a maximum interval of about -// 24 hours. Cancelling ctx will cancel retries and shut down -// any goroutines spawned by ManageAsync. -func (cfg *Config) ManageAsync(ctx context.Context, domainNames []string) error { - return cfg.manageAll(ctx, domainNames, true) -} - func (cfg *Config) manageAll(ctx context.Context, domainNames []string, async bool) error { if ctx == nil { ctx = context.Background() @@ -863,20 +863,28 @@ func (cfg *Config) RevokeCert(ctx context.Context, domain string, reason int, in return nil } -// TLSConfig is an opinionated method that returns a -// recommended, modern TLS configuration that can be -// used to configure TLS listeners, which also supports -// the TLS-ALPN challenge and serves up certificates -// managed by cfg. +// TLSConfig is an opinionated method that returns a recommended, modern +// TLS configuration that can be used to configure TLS listeners. Aside +// from safe, modern defaults, this method sets two critical fields on the +// TLS config which are required to enable automatic certificate +// management: GetCertificate and NextProtos. +// +// The GetCertificate field is necessary to get certificates from memory +// or storage, including both manual and automated certificates. You +// should only change this field if you know what you are doing. // -// Unlike the package TLS() function, this method does -// not, by itself, enable certificate management for -// any domain names. +// The NextProtos field is pre-populated with a special value to enable +// solving the TLS-ALPN ACME challenge. Because this method does not +// assume any particular protocols after the TLS handshake is completed, +// you will likely need to customize the NextProtos field by prepending +// your application's protocols to the slice. For example, to serve +// HTTP, you will need to prepend "h2" and "http/1.1" values. Be sure to +// leave the acmez.ACMETLS1Protocol value intact, however, or TLS-ALPN +// challenges will fail (which may be acceptable if you are not using +// ACME, or specifically, the TLS-ALPN challenge). // -// Feel free to further customize the returned tls.Config, -// but do not mess with the GetCertificate or NextProtos -// fields unless you know what you're doing, as they're -// necessary to solve the TLS-ALPN challenge. +// Unlike the package TLS() function, this method does not, by itself, +// enable certificate management for any domain names. func (cfg *Config) TLSConfig() *tls.Config { return &tls.Config{ // these two fields necessary for TLS-ALPN challenge diff --git a/vendor/github.com/caddyserver/certmagic/crypto.go b/vendor/github.com/caddyserver/certmagic/crypto.go index a705cdde2cdd..ee4671f5f59b 100644 --- a/vendor/github.com/caddyserver/certmagic/crypto.go +++ b/vendor/github.com/caddyserver/certmagic/crypto.go @@ -72,6 +72,10 @@ func encodePrivateKey(key crypto.PrivateKey) ([]byte, error) { func decodePrivateKey(keyPEMBytes []byte) (crypto.Signer, error) { keyBlockDER, _ := pem.Decode(keyPEMBytes) + if keyBlockDER == nil { + return nil, fmt.Errorf("failed to decode PEM block containing private key") + } + if keyBlockDER.Type != "PRIVATE KEY" && !strings.HasSuffix(keyBlockDER.Type, " PRIVATE KEY") { return nil, fmt.Errorf("unknown PEM header %q", keyBlockDER.Type) } @@ -142,14 +146,14 @@ func (cfg *Config) saveCertResource(issuer Issuer, cert CertificateResource) err certKey := cert.NamesKey() all := []keyValue{ - { - key: StorageKeys.SiteCert(issuerKey, certKey), - value: cert.CertificatePEM, - }, { key: StorageKeys.SitePrivateKey(issuerKey, certKey), value: cert.PrivateKeyPEM, }, + { + key: StorageKeys.SiteCert(issuerKey, certKey), + value: cert.CertificatePEM, + }, { key: StorageKeys.SiteMeta(issuerKey, certKey), value: metaBytes, diff --git a/vendor/github.com/caddyserver/certmagic/go.mod b/vendor/github.com/caddyserver/certmagic/go.mod index 42665bf1b320..a9267715b640 100644 --- a/vendor/github.com/caddyserver/certmagic/go.mod +++ b/vendor/github.com/caddyserver/certmagic/go.mod @@ -3,10 +3,10 @@ module github.com/caddyserver/certmagic go 1.14 require ( - github.com/klauspost/cpuid/v2 v2.0.6 + github.com/klauspost/cpuid/v2 v2.0.9 github.com/libdns/libdns v0.2.1 - github.com/mholt/acmez v0.1.3 - github.com/miekg/dns v1.1.42 + github.com/mholt/acmez v1.0.1 + github.com/miekg/dns v1.1.43 go.uber.org/zap v1.17.0 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/net v0.0.0-20210525063256-abc453219eb5 diff --git a/vendor/github.com/caddyserver/certmagic/go.sum b/vendor/github.com/caddyserver/certmagic/go.sum index 1251ac84f0ce..d9512a8062a8 100644 --- a/vendor/github.com/caddyserver/certmagic/go.sum +++ b/vendor/github.com/caddyserver/certmagic/go.sum @@ -4,8 +4,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= -github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -13,10 +13,10 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= -github.com/mholt/acmez v0.1.3 h1:J7MmNIk4Qf9b8mAGqAh4XkNeowv3f1zW816yf4zt7Qk= -github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= -github.com/miekg/dns v1.1.42 h1:gWGe42RGaIqXQZ+r3WUGEKBEtvPHY2SXo4dqixDNxuY= -github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mholt/acmez v1.0.1 h1:J7uquHOKEmo71UDnVApy1sSLA0oF/r+NtVrNzMKKA9I= +github.com/mholt/acmez v1.0.1/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= +github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/vendor/github.com/caddyserver/certmagic/handshake.go b/vendor/github.com/caddyserver/certmagic/handshake.go index 5506719c2b2b..e57a3d168301 100644 --- a/vendor/github.com/caddyserver/certmagic/handshake.go +++ b/vendor/github.com/caddyserver/certmagic/handshake.go @@ -125,23 +125,6 @@ func (cfg *Config) getCertificate(hello *tls.ClientHelloInfo) (cert Certificate, return } } - - // check the certCache directly to see if the SNI name is - // already the key of the certificate it wants; this implies - // that the SNI can contain the hash of a specific cert - // (chain) it wants and we will still be able to serve it up - // (this behavior, by the way, could be controversial as to - // whether it complies with RFC 6066 about SNI, but I think - // it does, soooo...) - // (this is how we solved the former ACME TLS-SNI challenge) - cfg.certCache.mu.RLock() - directCert, ok := cfg.certCache.cache[name] - cfg.certCache.mu.RUnlock() - if ok { - cert = directCert - matched = true - return - } } // otherwise, we're bingo on ammo; see issues @@ -162,18 +145,48 @@ func (cfg *Config) getCertificate(hello *tls.ClientHelloInfo) (cert Certificate, // then all certificates in the cache will be passed in // for the cfg.CertSelection to make the final decision. func (cfg *Config) selectCert(hello *tls.ClientHelloInfo, name string) (Certificate, bool) { + logger := loggerNamed(cfg.Logger, "handshake") choices := cfg.certCache.getAllMatchingCerts(name) if len(choices) == 0 { if cfg.CertSelection == nil { + if logger != nil { + logger.Debug("no matching certificates and no custom selection logic", zap.String("identifier", name)) + } return Certificate{}, false } + if logger != nil { + logger.Debug("no matching certificate; will choose from all certificates", zap.String("identifier", name)) + } choices = cfg.certCache.getAllCerts() } + if logger != nil { + logger.Debug("choosing certificate", + zap.String("identifier", name), + zap.Int("num_choices", len(choices))) + } if cfg.CertSelection == nil { cert, err := DefaultCertificateSelector(hello, choices) + if logger != nil { + logger.Debug("default certificate selection results", + zap.Error(err), + zap.String("identifier", name), + zap.Strings("subjects", cert.Names), + zap.Bool("managed", cert.managed), + zap.String("issuer_key", cert.issuerKey), + zap.String("hash", cert.hash)) + } return cert, err == nil } cert, err := cfg.CertSelection.SelectCertificate(hello, choices) + if logger != nil { + logger.Debug("custom certificate selection results", + zap.Error(err), + zap.String("identifier", name), + zap.Strings("subjects", cert.Names), + zap.Bool("managed", cert.managed), + zap.String("issuer_key", cert.issuerKey), + zap.String("hash", cert.hash)) + } return cert, err == nil } @@ -213,28 +226,54 @@ func DefaultCertificateSelector(hello *tls.ClientHelloInfo, choices []Certificat // // This function is safe for concurrent use. func (cfg *Config) getCertDuringHandshake(hello *tls.ClientHelloInfo, loadIfNecessary, obtainIfNecessary bool) (Certificate, error) { - log := loggerNamed(cfg.Logger, "on_demand") + log := loggerNamed(cfg.Logger, "handshake") // First check our in-memory cache to see if we've already loaded it cert, matched, defaulted := cfg.getCertificate(hello) if matched { + if log != nil { + log.Debug("matched certificate in cache", + zap.Strings("subjects", cert.Names), + zap.Bool("managed", cert.managed), + zap.Time("expiration", cert.Leaf.NotAfter), + zap.String("hash", cert.hash)) + } if cert.managed && cfg.OnDemand != nil && obtainIfNecessary { // It's been reported before that if the machine goes to sleep (or // suspends the process) that certs which are already loaded into // memory won't get renewed in the background, so we need to check // expiry on each handshake too, sigh: // https://caddy.community/t/local-certificates-not-renewing-on-demand/9482 - return cfg.optionalMaintenance(log, cert, hello) + return cfg.optionalMaintenance(loggerNamed(cfg.Logger, "on_demand"), cert, hello) } return cert, nil } name := cfg.getNameFromClientHello(hello) - // If OnDemand is enabled, then we might be able to load or - // obtain a needed certificate - if cfg.OnDemand != nil && loadIfNecessary { + // We might be able to load or obtain a needed certificate. Load from + // storage if OnDemand is enabled, or if there is the possibility that + // a statically-managed cert was evicted from a full cache. + cfg.certCache.mu.RLock() + cacheSize := len(cfg.certCache.cache) + cfg.certCache.mu.RUnlock() + + // A cert might have still been evicted from the cache even if the cache + // is no longer completely full; this happens if the newly-loaded cert is + // itself evicted (perhaps due to being expired or unmanaged at this point). + // Hence, we use an "almost full" metric to allow for the cache to not be + // perfectly full while still being able to load needed certs from storage. + // See https://caddy.community/t/error-tls-alert-internal-error-592-again/13272 + // and caddyserver/caddy#4320. + cacheAlmostFull := float64(cacheSize) >= (float64(cfg.certCache.options.Capacity) * .9) + loadDynamically := cfg.OnDemand != nil || cacheAlmostFull + + if loadDynamically && loadIfNecessary { // Then check to see if we have one on disk + // TODO: As suggested here, https://caddy.community/t/error-tls-alert-internal-error-592-again/13272/30?u=matt, + // it might be a good idea to check with the DecisionFunc or allowlist first before even loading the certificate + // from storage, since if we can't renew it, why should we even try serving it (it will just get evicted after + // we get a return value of false anyway)? loadedCert, err := cfg.CacheManagedCertificate(name) if _, ok := err.(ErrNotExist); ok { // If no exact match, try a wildcard variant, which is something we can still use @@ -243,6 +282,13 @@ func (cfg *Config) getCertDuringHandshake(hello *tls.ClientHelloInfo, loadIfNece loadedCert, err = cfg.CacheManagedCertificate(strings.Join(labels, ".")) } if err == nil { + if log != nil { + log.Debug("loaded certificate from storage", + zap.Strings("subjects", loadedCert.Names), + zap.Bool("managed", loadedCert.managed), + zap.Time("expiration", loadedCert.Leaf.NotAfter), + zap.String("hash", loadedCert.hash)) + } loadedCert, err = cfg.handshakeMaintenance(hello, loadedCert) if err != nil { if log != nil { @@ -253,7 +299,7 @@ func (cfg *Config) getCertDuringHandshake(hello *tls.ClientHelloInfo, loadIfNece } return loadedCert, nil } - if obtainIfNecessary { + if cfg.OnDemand != nil && obtainIfNecessary { // By this point, we need to ask the CA for a certificate return cfg.obtainOnDemandCertificate(hello) } @@ -261,9 +307,28 @@ func (cfg *Config) getCertDuringHandshake(hello *tls.ClientHelloInfo, loadIfNece // Fall back to the default certificate if there is one if defaulted { + if log != nil { + log.Debug("fell back to default certificate", + zap.Strings("subjects", cert.Names), + zap.Bool("managed", cert.managed), + zap.Time("expiration", cert.Leaf.NotAfter), + zap.String("hash", cert.hash)) + } return cert, nil } + if log != nil { + log.Debug("no certificate matching TLS ClientHello", + zap.String("server_name", hello.ServerName), + zap.String("remote", hello.Conn.RemoteAddr().String()), + zap.String("identifier", name), + zap.Uint16s("cipher_suites", hello.CipherSuites), + zap.Float64("cert_cache_fill", float64(cacheSize)/float64(cfg.certCache.options.Capacity)), // may be approximate! because we are not within the lock + zap.Bool("load_if_necessary", loadIfNecessary), + zap.Bool("obtain_if_necessary", obtainIfNecessary), + zap.Bool("on_demand", cfg.OnDemand != nil)) + } + return Certificate{}, fmt.Errorf("no certificate available for '%s'", name) } @@ -371,7 +436,8 @@ func (cfg *Config) obtainOnDemandCertificate(hello *tls.ClientHelloInfo) (Certif } // TODO: use a proper context; we use one with timeout because retries are enabled because interactive is false - ctx, cancel := context.WithTimeout(context.TODO(), 90*time.Second) + // (timeout duration is based on https://caddy.community/t/zerossl-dns-challenge-failing-often-route53-plugin/13822/24?u=matt) + ctx, cancel := context.WithTimeout(context.TODO(), 180*time.Second) defer cancel() // Obtain the certificate @@ -459,7 +525,7 @@ func (cfg *Config) renewDynamicCertificate(hello *tls.ClientHelloInfo, currentCe // renewing it, so we might as well serve what we have without blocking if log != nil { log.Debug("certificate expires soon but is already being renewed; serving current certificate", - zap.Strings("identifiers", currentCert.Names), + zap.Strings("subjects", currentCert.Names), zap.Duration("remaining", timeLeft)) } return currentCert, nil @@ -470,7 +536,7 @@ func (cfg *Config) renewDynamicCertificate(hello *tls.ClientHelloInfo, currentCe if log != nil { log.Debug("certificate has expired, but is already being renewed; waiting for renewal to complete", - zap.Strings("identifiers", currentCert.Names), + zap.Strings("subjects", currentCert.Names), zap.Time("expired", currentCert.Leaf.NotAfter)) } @@ -501,7 +567,7 @@ func (cfg *Config) renewDynamicCertificate(hello *tls.ClientHelloInfo, currentCe if log != nil { log.Info("attempting certificate renewal", zap.String("server_name", name), - zap.Strings("identifiers", currentCert.Names), + zap.Strings("subjects", currentCert.Names), zap.Time("expiration", currentCert.Leaf.NotAfter), zap.Duration("remaining", timeLeft)) } diff --git a/vendor/github.com/caddyserver/certmagic/solvers.go b/vendor/github.com/caddyserver/certmagic/solvers.go index 8c7199581430..8cdaeaf839be 100644 --- a/vendor/github.com/caddyserver/certmagic/solvers.go +++ b/vendor/github.com/caddyserver/certmagic/solvers.go @@ -32,6 +32,7 @@ import ( "github.com/libdns/libdns" "github.com/mholt/acmez" "github.com/mholt/acmez/acme" + "github.com/miekg/dns" ) // httpSolver solves the HTTP challenge. It must be @@ -131,10 +132,12 @@ func (s *tlsALPNSolver) Present(ctx context.Context, chal acme.Challenge) error if err != nil { return err } + + key := challengeKey(chal) activeChallengesMu.Lock() - chalData := activeChallenges[chal.Identifier.Value] + chalData := activeChallenges[key] chalData.data = cert - activeChallenges[chal.Identifier.Value] = chalData + activeChallenges[key] = chalData activeChallengesMu.Unlock() // the rest of this function increments the @@ -215,10 +218,6 @@ func (*tlsALPNSolver) handleConn(conn net.Conn) { // CleanUp removes the challenge certificate from the cache, and if // it is the last one to finish, stops the TLS server. func (s *tlsALPNSolver) CleanUp(ctx context.Context, chal acme.Challenge) error { - s.config.certCache.mu.Lock() - delete(s.config.certCache.cache, tlsALPNCertKeyName(chal.Identifier.Value)) - s.config.certCache.mu.Unlock() - solversMu.Lock() defer solversMu.Unlock() si := getSolverInfo(s.address) @@ -236,14 +235,6 @@ func (s *tlsALPNSolver) CleanUp(ctx context.Context, chal acme.Challenge) error return nil } -// tlsALPNCertKeyName returns the key to use when caching a cert -// for use with the TLS-ALPN ACME challenge. It is simply to help -// avoid conflicts (although at time of writing, there shouldn't -// be, since the cert cache is keyed by hash of certificate chain). -func tlsALPNCertKeyName(sniName string) string { - return sniName + ":acme-tls-alpn" -} - // DNS01Solver is a type that makes libdns providers usable // as ACME dns-01 challenge solvers. // See https://github.com/libdns/libdns @@ -478,7 +469,7 @@ func (dhs distributedSolver) Present(ctx context.Context, chal acme.Challenge) e return err } - err = dhs.storage.Store(dhs.challengeTokensKey(chal.Identifier.Value), infoBytes) + err = dhs.storage.Store(dhs.challengeTokensKey(challengeKey(chal)), infoBytes) if err != nil { return err } @@ -501,7 +492,7 @@ func (dhs distributedSolver) Wait(ctx context.Context, challenge acme.Challenge) // CleanUp invokes the underlying solver's CleanUp method // and also cleans up any assets saved to storage. func (dhs distributedSolver) CleanUp(ctx context.Context, chal acme.Challenge) error { - err := dhs.storage.Delete(dhs.challengeTokensKey(chal.Identifier.Value)) + err := dhs.storage.Delete(dhs.challengeTokensKey(challengeKey(chal))) if err != nil { return err } @@ -648,6 +639,18 @@ type Challenge struct { data interface{} } +// challengeKey returns the map key for a given challenge; it is the identifier +// unless it is an IP address using the TLS-ALPN challenge. +func challengeKey(chal acme.Challenge) string { + if chal.Type == acme.ChallengeTypeTLSALPN01 && chal.Identifier.Type == "ip" { + reversed, err := dns.ReverseAddr(chal.Identifier.Value) + if err == nil { + return reversed[:len(reversed)-1] // strip off '.' + } + } + return chal.Identifier.Value +} + // solverWrapper should be used to wrap all challenge solvers so that // we can add the challenge info to memory; this makes challenges globally // solvable by a single HTTP or TLS server even if multiple servers with @@ -656,7 +659,7 @@ type solverWrapper struct{ acmez.Solver } func (sw solverWrapper) Present(ctx context.Context, chal acme.Challenge) error { activeChallengesMu.Lock() - activeChallenges[chal.Identifier.Value] = Challenge{Challenge: chal} + activeChallenges[challengeKey(chal)] = Challenge{Challenge: chal} activeChallengesMu.Unlock() return sw.Solver.Present(ctx, chal) } @@ -670,7 +673,7 @@ func (sw solverWrapper) Wait(ctx context.Context, chal acme.Challenge) error { func (sw solverWrapper) CleanUp(ctx context.Context, chal acme.Challenge) error { activeChallengesMu.Lock() - delete(activeChallenges, chal.Identifier.Value) + delete(activeChallenges, challengeKey(chal)) activeChallengesMu.Unlock() return sw.Solver.CleanUp(ctx, chal) } diff --git a/vendor/github.com/mholt/acmez/acme/certificate.go b/vendor/github.com/mholt/acmez/acme/certificate.go index a778280802bb..42bbba07e3df 100644 --- a/vendor/github.com/mholt/acmez/acme/certificate.go +++ b/vendor/github.com/mholt/acmez/acme/certificate.go @@ -111,7 +111,7 @@ func (c *Client) GetCertificateChain(ctx context.Context, account Account, certU // heuristics to decide which is optimal." §7.4.2 alternates := extractLinks(resp, "alternate") for _, altURL := range alternates { - resp, err = addChain(altURL) + _, err = addChain(altURL) if err != nil { return nil, fmt.Errorf("retrieving alternate certificate chain at %s: %w", altURL, err) } diff --git a/vendor/github.com/mholt/acmez/acme/http.go b/vendor/github.com/mholt/acmez/acme/http.go index a910d57e562d..63688013e943 100644 --- a/vendor/github.com/mholt/acmez/acme/http.go +++ b/vendor/github.com/mholt/acmez/acme/http.go @@ -117,8 +117,7 @@ func (c *Client) httpPostJWS(ctx context.Context, privateKey crypto.Signer, break } - return resp, fmt.Errorf("request to %s failed after %d attempts: %w", - endpoint, attempts, err) + return resp, fmt.Errorf("attempt %d: %s: %w", attempts, endpoint, err) } // httpReq robustly performs an HTTP request using the given method to the given endpoint, honoring @@ -272,8 +271,8 @@ func (c *Client) doHTTPRequest(req *http.Request, buf *bytes.Buffer) (resp *http zap.String("method", req.Method), zap.String("url", req.URL.String()), zap.Reflect("headers", req.Header), - zap.Int("status_code", resp.StatusCode), - zap.Reflect("response_headers", resp.Header)) + zap.Reflect("response_headers", resp.Header), + zap.Int("status_code", resp.StatusCode)) } // "The server MUST include a Replay-Nonce header field diff --git a/vendor/github.com/mholt/acmez/acme/problem.go b/vendor/github.com/mholt/acmez/acme/problem.go index 98fdb0095885..c5f1124649d5 100644 --- a/vendor/github.com/mholt/acmez/acme/problem.go +++ b/vendor/github.com/mholt/acmez/acme/problem.go @@ -14,7 +14,11 @@ package acme -import "fmt" +import ( + "fmt" + + "go.uber.org/zap/zapcore" +) // Problem carries the details of an error from HTTP APIs as // defined in RFC 7807: https://tools.ietf.org/html/rfc7807 @@ -77,6 +81,9 @@ func (p Problem) Error() string { if len(p.Subproblems) > 0 { for _, v := range p.Subproblems { s += fmt.Sprintf(", problem %q: %s", v.Type, v.Detail) + if v.Identifier.Type != "" || v.Identifier.Value != "" { + s += fmt.Sprintf(" (%s_identifier=%s)", v.Identifier.Type, v.Identifier.Value) + } } } if p.Instance != "" { @@ -85,6 +92,17 @@ func (p Problem) Error() string { return s } +// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface. +// This allows problems to be serialized by the zap logger. +func (p Problem) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("type", p.Type) + enc.AddString("title", p.Title) + enc.AddString("detail", p.Detail) + enc.AddString("instance", p.Instance) + enc.AddArray("subproblems", loggableSubproblems(p.Subproblems)) + return nil +} + // Subproblem describes a more specific error in a problem according to // RFC 8555 §6.7.1: "An ACME problem document MAY contain the // 'subproblems' field, containing a JSON array of problem documents, @@ -97,6 +115,26 @@ type Subproblem struct { Identifier Identifier `json:"identifier,omitempty"` } +// MarshalLogObject satisfies the zapcore.ObjectMarshaler interface. +// This allows subproblems to be serialized by the zap logger. +func (sp Subproblem) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("identifier_type", sp.Identifier.Type) + enc.AddString("identifier", sp.Identifier.Value) + enc.AddObject("subproblem", sp.Problem) + return nil +} + +type loggableSubproblems []Subproblem + +// MarshalLogArray satisfies the zapcore.ArrayMarshaler interface. +// This allows a list of subproblems to be serialized by the zap logger. +func (ls loggableSubproblems) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, sp := range ls { + enc.AppendObject(sp) + } + return nil +} + // Standard token values for the "type" field of problems, as defined // in RFC 8555 §6.7: https://tools.ietf.org/html/rfc8555#section-6.7 // diff --git a/vendor/github.com/mholt/acmez/client.go b/vendor/github.com/mholt/acmez/client.go index a4d0446bb882..3add982cac5b 100644 --- a/vendor/github.com/mholt/acmez/client.go +++ b/vendor/github.com/mholt/acmez/client.go @@ -134,16 +134,23 @@ func (c *Client) ObtainCertificateUsingCSR(ctx context.Context, account acme.Acc // for some errors, we can retry with different challenge types var problem acme.Problem if errors.As(err, &problem) { - authz := problem.Resource.(acme.Authorization) + authz, haveAuthz := problem.Resource.(acme.Authorization) if c.Logger != nil { - c.Logger.Error("validating authorization", - zap.String("identifier", authz.IdentifierValue()), - zap.Error(err), + l := c.Logger + if haveAuthz { + l = l.With(zap.String("identifier", authz.IdentifierValue())) + } + l.Error("validating authorization", + zap.Object("problem", problem), zap.String("order", order.Location), zap.Int("attempt", attempt), zap.Int("max_attempts", maxAttempts)) } - err = fmt.Errorf("solving challenge: %s: %w", authz.IdentifierValue(), err) + errStr := "solving challenge" + if haveAuthz { + errStr += ": " + authz.IdentifierValue() + } + err = fmt.Errorf("%s: %w", errStr, err) if errors.As(err, &retryableErr{}) { continue } @@ -505,9 +512,7 @@ func (c *Client) pollAuthorization(ctx context.Context, account acme.Account, au c.Logger.Error("challenge failed", zap.String("identifier", authz.IdentifierValue()), zap.String("challenge_type", authz.currentChallenge.Type), - zap.Int("status_code", problem.Status), - zap.String("problem_type", problem.Type), - zap.String("error", problem.Detail)) + zap.Object("problem", problem)) } failedChallengeTypes.rememberFailedChallenge(authz) diff --git a/vendor/modules.txt b/vendor/modules.txt index dfc355e9040a..23a59c18ac12 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -197,7 +197,7 @@ github.com/boombuler/barcode/utils # github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b ## explicit github.com/bradfitz/gomemcache/memcache -# github.com/caddyserver/certmagic v0.14.1 +# github.com/caddyserver/certmagic v0.15.2 ## explicit github.com/caddyserver/certmagic # github.com/cespare/xxhash/v2 v2.1.1 @@ -606,7 +606,7 @@ github.com/mattn/go-runewidth github.com/mattn/go-sqlite3 # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil -# github.com/mholt/acmez v0.1.3 +# github.com/mholt/acmez v1.0.1 github.com/mholt/acmez github.com/mholt/acmez/acme # github.com/mholt/archiver/v3 v3.5.0 @@ -617,7 +617,6 @@ github.com/mholt/archiver/v3 github.com/microcosm-cc/bluemonday github.com/microcosm-cc/bluemonday/css # github.com/miekg/dns v1.1.43 -## explicit github.com/miekg/dns # github.com/minio/md5-simd v1.1.2 ## explicit From 549fd03c0e9aa19850c8271cbd7cd62bbc884b34 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 1 Jan 2022 17:43:44 +0800 Subject: [PATCH 04/22] Upgrade certmagic from v0.14.1 to v0.15.2 (#18138) From 7db2f110adbd020e70c56497306cfbda8806d109 Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Sat, 1 Jan 2022 15:12:25 +0100 Subject: [PATCH 05/22] Add API to get issue/pull comments and events (timeline) (#17403) * Add API to get issue/pull comments and events (timeline) Adds an API to get both comments and events in one endpoint with all required data. Closes go-gitea/gitea#13250 * Fix swagger * Don't show code comments (use review api instead) * fmt * Fix comment * Time -> TrackedTime * Use var directly * Add logger * Fix lint * Fix test * Add comments * fmt * [test] get issue directly by ID * Update test * Add description for changed refs * Fix build issues + lint * Fix build * Use string enums * Update swagger * Support `page` and `limit` params * fmt + swagger * Use global slices Co-authored-by: zeripath Co-authored-by: Lunny Xiao --- integrations/api_comment_test.go | 22 +++ models/issue_comment.go | 41 ++++++ modules/convert/issue_comment.go | 143 +++++++++++++++++++ modules/references/references.go | 11 ++ modules/structs/issue_comment.go | 45 ++++++ routers/api/v1/api.go | 1 + routers/api/v1/repo/issue_comment.go | 111 +++++++++++++++ routers/api/v1/swagger/issue.go | 7 + templates/swagger/v1_json.tmpl | 196 +++++++++++++++++++++++++++ 9 files changed, 577 insertions(+) diff --git a/integrations/api_comment_test.go b/integrations/api_comment_test.go index 0c3ac2ae5b56..4c4c6308eeab 100644 --- a/integrations/api_comment_test.go +++ b/integrations/api_comment_test.go @@ -180,3 +180,25 @@ func TestAPIDeleteComment(t *testing.T) { unittest.AssertNotExistsBean(t, &models.Comment{ID: comment.ID}) } + +func TestAPIListIssueTimeline(t *testing.T) { + defer prepareTestEnv(t)() + + // load comment + issue := unittest.AssertExistsAndLoadBean(t, &models.Issue{ID: 1}).(*models.Issue) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: issue.RepoID}).(*repo_model.Repository) + repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User) + + // make request + session := loginUser(t, repoOwner.Name) + req := NewRequestf(t, "GET", "/api/v1/repos/%s/%s/issues/%d/timeline", + repoOwner.Name, repo.Name, issue.Index) + resp := session.MakeRequest(t, req, http.StatusOK) + + // check if lens of list returned by API and + // lists extracted directly from DB are the same + var comments []*api.TimelineComment + DecodeJSON(t, resp, &comments) + expectedCount := unittest.GetCount(t, &models.Comment{IssueID: issue.ID}) + assert.EqualValues(t, expectedCount, len(comments)) +} diff --git a/models/issue_comment.go b/models/issue_comment.go index 360a212a23b2..9a6f4247b082 100644 --- a/models/issue_comment.go +++ b/models/issue_comment.go @@ -110,6 +110,47 @@ const ( CommentTypeChangeIssueRef ) +var commentStrings = []string{ + "comment", + "reopen", + "close", + "issue_ref", + "commit_ref", + "comment_ref", + "pull_ref", + "label", + "milestone", + "assignees", + "change_title", + "delete_branch", + "start_tracking", + "stop_tracking", + "add_time_manual", + "cancel_tracking", + "added_deadline", + "modified_deadline", + "removed_deadline", + "add_dependency", + "remove_dependency", + "code", + "review", + "lock", + "unlock", + "change_target_branch", + "delete_time_manual", + "review_request", + "merge_pull", + "pull_push", + "project", + "project_board", + "dismiss_review", + "change_issue_ref", +} + +func (t CommentType) String() string { + return commentStrings[t] +} + // RoleDescriptor defines comment tag type type RoleDescriptor int diff --git a/modules/convert/issue_comment.go b/modules/convert/issue_comment.go index 1610b9f0d88c..caba2b506e43 100644 --- a/modules/convert/issue_comment.go +++ b/modules/convert/issue_comment.go @@ -6,6 +6,9 @@ package convert import ( "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" api "code.gitea.io/gitea/modules/structs" ) @@ -22,3 +25,143 @@ func ToComment(c *models.Comment) *api.Comment { Updated: c.UpdatedUnix.AsTime(), } } + +// ToTimelineComment converts a models.Comment to the api.TimelineComment format +func ToTimelineComment(c *models.Comment, doer *user_model.User) *api.TimelineComment { + err := c.LoadMilestone() + if err != nil { + log.Error("LoadMilestone: %v", err) + return nil + } + + err = c.LoadAssigneeUserAndTeam() + if err != nil { + log.Error("LoadAssigneeUserAndTeam: %v", err) + return nil + } + + err = c.LoadResolveDoer() + if err != nil { + log.Error("LoadResolveDoer: %v", err) + return nil + } + + err = c.LoadDepIssueDetails() + if err != nil { + log.Error("LoadDepIssueDetails: %v", err) + return nil + } + + err = c.LoadTime() + if err != nil { + log.Error("LoadTime: %v", err) + return nil + } + + err = c.LoadLabel() + if err != nil { + log.Error("LoadLabel: %v", err) + return nil + } + + comment := &api.TimelineComment{ + ID: c.ID, + Type: c.Type.String(), + Poster: ToUser(c.Poster, nil), + HTMLURL: c.HTMLURL(), + IssueURL: c.IssueURL(), + PRURL: c.PRURL(), + Body: c.Content, + Created: c.CreatedUnix.AsTime(), + Updated: c.UpdatedUnix.AsTime(), + + OldProjectID: c.OldProjectID, + ProjectID: c.ProjectID, + + OldTitle: c.OldTitle, + NewTitle: c.NewTitle, + + OldRef: c.OldRef, + NewRef: c.NewRef, + + RefAction: c.RefAction.String(), + RefCommitSHA: c.CommitSHA, + + ReviewID: c.ReviewID, + + RemovedAssignee: c.RemovedAssignee, + } + + if c.OldMilestone != nil { + comment.OldMilestone = ToAPIMilestone(c.OldMilestone) + } + if c.Milestone != nil { + comment.Milestone = ToAPIMilestone(c.Milestone) + } + + if c.Time != nil { + comment.TrackedTime = ToTrackedTime(c.Time) + } + + if c.RefIssueID != 0 { + issue, err := models.GetIssueByID(c.RefIssueID) + if err != nil { + log.Error("GetIssueByID(%d): %v", c.RefIssueID, err) + return nil + } + comment.RefIssue = ToAPIIssue(issue) + } + + if c.RefCommentID != 0 { + com, err := models.GetCommentByID(c.RefCommentID) + if err != nil { + log.Error("GetCommentByID(%d): %v", c.RefCommentID, err) + return nil + } + err = com.LoadPoster() + if err != nil { + log.Error("LoadPoster: %v", err) + return nil + } + comment.RefComment = ToComment(com) + } + + if c.Label != nil { + var org *user_model.User + var repo *repo_model.Repository + if c.Label.BelongsToOrg() { + var err error + org, err = user_model.GetUserByID(c.Label.OrgID) + if err != nil { + log.Error("GetUserByID(%d): %v", c.Label.OrgID, err) + return nil + } + } + if c.Label.BelongsToRepo() { + var err error + repo, err = repo_model.GetRepositoryByID(c.Label.RepoID) + if err != nil { + log.Error("GetRepositoryByID(%d): %v", c.Label.RepoID, err) + return nil + } + } + comment.Label = ToLabel(c.Label, repo, org) + } + + if c.Assignee != nil { + comment.Assignee = ToUser(c.Assignee, nil) + } + if c.AssigneeTeam != nil { + comment.AssigneeTeam = ToTeam(c.AssigneeTeam) + } + + if c.ResolveDoer != nil { + comment.ResolveDoer = ToUser(c.ResolveDoer, nil) + } + + if c.DependentIssue != nil { + comment.DependentIssue = ToAPIIssue(c.DependentIssue) + } + + return comment +} diff --git a/modules/references/references.go b/modules/references/references.go index cfc01cd4c013..74837b8553f6 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -49,6 +49,13 @@ var ( giteaHostInit sync.Once giteaHost string giteaIssuePullPattern *regexp.Regexp + + actionStrings = []string{ + "none", + "closes", + "reopens", + "neutered", + } ) // XRefAction represents the kind of effect a cross reference has once is resolved @@ -65,6 +72,10 @@ const ( XRefActionNeutered // 3 ) +func (a XRefAction) String() string { + return actionStrings[a] +} + // IssueReference contains an unverified cross-reference to a local issue or pull request type IssueReference struct { Index int64 diff --git a/modules/structs/issue_comment.go b/modules/structs/issue_comment.go index 0c8ac20017fb..e13ec05d0185 100644 --- a/modules/structs/issue_comment.go +++ b/modules/structs/issue_comment.go @@ -35,3 +35,48 @@ type EditIssueCommentOption struct { // required: true Body string `json:"body" binding:"Required"` } + +// TimelineComment represents a timeline comment (comment of any type) on a commit or issue +type TimelineComment struct { + ID int64 `json:"id"` + Type string `json:"type"` + + HTMLURL string `json:"html_url"` + PRURL string `json:"pull_request_url"` + IssueURL string `json:"issue_url"` + Poster *User `json:"user"` + Body string `json:"body"` + // swagger:strfmt date-time + Created time.Time `json:"created_at"` + // swagger:strfmt date-time + Updated time.Time `json:"updated_at"` + + OldProjectID int64 `json:"old_project_id"` + ProjectID int64 `json:"project_id"` + OldMilestone *Milestone `json:"old_milestone"` + Milestone *Milestone `json:"milestone"` + TrackedTime *TrackedTime `json:"tracked_time"` + OldTitle string `json:"old_title"` + NewTitle string `json:"new_title"` + OldRef string `json:"old_ref"` + NewRef string `json:"new_ref"` + + RefIssue *Issue `json:"ref_issue"` + RefComment *Comment `json:"ref_comment"` + RefAction string `json:"ref_action"` + // commit SHA where issue/PR was referenced + RefCommitSHA string `json:"ref_commit_sha"` + + ReviewID int64 `json:"review_id"` + + Label *Label `json:"label"` + + Assignee *User `json:"assignee"` + AssigneeTeam *Team `json:"assignee_team"` + // whether the assignees were removed or added + RemovedAssignee bool `json:"removed_assignee"` + + ResolveDoer *User `json:"resolve_doer"` + + DependentIssue *Issue `json:"dependent_issue"` +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index c587907d4b2b..7a2347650a05 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -842,6 +842,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { m.Combo("/{id}", reqToken()).Patch(bind(api.EditIssueCommentOption{}), repo.EditIssueCommentDeprecated). Delete(repo.DeleteIssueCommentDeprecated) }) + m.Get("/timeline", repo.ListIssueCommentsAndTimeline) m.Group("/labels", func() { m.Combo("").Get(repo.ListIssueLabels). Post(reqToken(), bind(api.IssueLabelsOption{}), repo.AddIssueLabels). diff --git a/routers/api/v1/repo/issue_comment.go b/routers/api/v1/repo/issue_comment.go index 13e7de46b1f1..b929cec3737c 100644 --- a/routers/api/v1/repo/issue_comment.go +++ b/routers/api/v1/repo/issue_comment.go @@ -10,6 +10,8 @@ import ( "net/http" "code.gitea.io/gitea/models" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" api "code.gitea.io/gitea/modules/structs" @@ -102,6 +104,115 @@ func ListIssueComments(ctx *context.APIContext) { ctx.JSON(http.StatusOK, &apiComments) } +// ListIssueCommentsAndTimeline list all the comments and events of an issue +func ListIssueCommentsAndTimeline(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/issues/{index}/timeline issue issueGetCommentsAndTimeline + // --- + // summary: List all comments and events on an issue + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: index + // in: path + // description: index of the issue + // type: integer + // format: int64 + // required: true + // - name: since + // in: query + // description: if provided, only comments updated since the specified time are returned. + // type: string + // format: date-time + // - name: page + // in: query + // description: page number of results to return (1-based) + // type: integer + // - name: limit + // in: query + // description: page size of results + // type: integer + // - name: before + // in: query + // description: if provided, only comments updated before the provided time are returned. + // type: string + // format: date-time + // responses: + // "200": + // "$ref": "#/responses/TimelineList" + + before, since, err := utils.GetQueryBeforeSince(ctx) + if err != nil { + ctx.Error(http.StatusUnprocessableEntity, "GetQueryBeforeSince", err) + return + } + issue, err := models.GetIssueByIndex(ctx.Repo.Repository.ID, ctx.ParamsInt64(":index")) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetRawIssueByIndex", err) + return + } + issue.Repo = ctx.Repo.Repository + + opts := &models.FindCommentsOptions{ + ListOptions: utils.GetListOptions(ctx), + IssueID: issue.ID, + Since: since, + Before: before, + Type: models.CommentTypeUnknown, + } + + comments, err := models.FindComments(opts) + if err != nil { + ctx.Error(http.StatusInternalServerError, "FindComments", err) + return + } + + if err := models.CommentList(comments).LoadPosters(); err != nil { + ctx.Error(http.StatusInternalServerError, "LoadPosters", err) + return + } + + var apiComments []*api.TimelineComment + for _, comment := range comments { + if comment.Type != models.CommentTypeCode && isXRefCommentAccessible(ctx.User, comment, issue.RepoID) { + comment.Issue = issue + apiComments = append(apiComments, convert.ToTimelineComment(comment, ctx.User)) + } + } + + ctx.SetTotalCountHeader(int64(len(apiComments))) + ctx.JSON(http.StatusOK, &apiComments) +} + +func isXRefCommentAccessible(user *user_model.User, c *models.Comment, issueRepoID int64) bool { + // Remove comments that the user has no permissions to see + if models.CommentTypeIsRef(c.Type) && c.RefRepoID != issueRepoID && c.RefRepoID != 0 { + var err error + // Set RefRepo for description in template + c.RefRepo, err = repo_model.GetRepositoryByID(c.RefRepoID) + if err != nil { + return false + } + perm, err := models.GetUserRepoPermission(c.RefRepo, user) + if err != nil { + return false + } + if !perm.CanReadIssuesOrPulls(c.RefIsPull) { + return false + } + } + return true +} + // ListRepoIssueComments returns all issue-comments for a repo func ListRepoIssueComments(ctx *context.APIContext) { // swagger:operation GET /repos/{owner}/{repo}/issues/comments issue issueGetRepoComments diff --git a/routers/api/v1/swagger/issue.go b/routers/api/v1/swagger/issue.go index 0f2f57202080..09e7077b200c 100644 --- a/routers/api/v1/swagger/issue.go +++ b/routers/api/v1/swagger/issue.go @@ -36,6 +36,13 @@ type swaggerResponseCommentList struct { Body []api.Comment `json:"body"` } +// TimelineList +// swagger:response TimelineList +type swaggerResponseTimelineList struct { + // in:body + Body []api.TimelineComment `json:"body"` +} + // Label // swagger:response Label type swaggerResponseLabel struct { diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 992cdf5bdac2..9438c41a29a6 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -6057,6 +6057,73 @@ } } }, + "/repos/{owner}/{repo}/issues/{index}/timeline": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "issue" + ], + "summary": "List all comments and events on an issue", + "operationId": "issueGetCommentsAndTimeline", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "integer", + "format": "int64", + "description": "index of the issue", + "name": "index", + "in": "path", + "required": true + }, + { + "type": "string", + "format": "date-time", + "description": "if provided, only comments updated since the specified time are returned.", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "page number of results to return (1-based)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "page size of results", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "format": "date-time", + "description": "if provided, only comments updated before the provided time are returned.", + "name": "before", + "in": "query" + } + ], + "responses": { + "200": { + "$ref": "#/responses/TimelineList" + } + } + } + }, "/repos/{owner}/{repo}/issues/{index}/times": { "get": { "produces": [ @@ -17396,6 +17463,126 @@ "format": "int64", "x-go-package": "code.gitea.io/gitea/modules/timeutil" }, + "TimelineComment": { + "description": "TimelineComment represents a timeline comment (comment of any type) on a commit or issue", + "type": "object", + "properties": { + "assignee": { + "$ref": "#/definitions/User" + }, + "assignee_team": { + "$ref": "#/definitions/Team" + }, + "body": { + "type": "string", + "x-go-name": "Body" + }, + "created_at": { + "type": "string", + "format": "date-time", + "x-go-name": "Created" + }, + "dependent_issue": { + "$ref": "#/definitions/Issue" + }, + "html_url": { + "type": "string", + "x-go-name": "HTMLURL" + }, + "id": { + "type": "integer", + "format": "int64", + "x-go-name": "ID" + }, + "issue_url": { + "type": "string", + "x-go-name": "IssueURL" + }, + "label": { + "$ref": "#/definitions/Label" + }, + "milestone": { + "$ref": "#/definitions/Milestone" + }, + "new_ref": { + "type": "string", + "x-go-name": "NewRef" + }, + "new_title": { + "type": "string", + "x-go-name": "NewTitle" + }, + "old_milestone": { + "$ref": "#/definitions/Milestone" + }, + "old_project_id": { + "type": "integer", + "format": "int64", + "x-go-name": "OldProjectID" + }, + "old_ref": { + "type": "string", + "x-go-name": "OldRef" + }, + "old_title": { + "type": "string", + "x-go-name": "OldTitle" + }, + "project_id": { + "type": "integer", + "format": "int64", + "x-go-name": "ProjectID" + }, + "pull_request_url": { + "type": "string", + "x-go-name": "PRURL" + }, + "ref_action": { + "type": "string", + "x-go-name": "RefAction" + }, + "ref_comment": { + "$ref": "#/definitions/Comment" + }, + "ref_commit_sha": { + "description": "commit SHA where issue/PR was referenced", + "type": "string", + "x-go-name": "RefCommitSHA" + }, + "ref_issue": { + "$ref": "#/definitions/Issue" + }, + "removed_assignee": { + "description": "whether the assignees were removed or added", + "type": "boolean", + "x-go-name": "RemovedAssignee" + }, + "resolve_doer": { + "$ref": "#/definitions/User" + }, + "review_id": { + "type": "integer", + "format": "int64", + "x-go-name": "ReviewID" + }, + "tracked_time": { + "$ref": "#/definitions/TrackedTime" + }, + "type": { + "type": "string", + "x-go-name": "Type" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "x-go-name": "Updated" + }, + "user": { + "$ref": "#/definitions/User" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "TopicName": { "description": "TopicName a list of repo topic names", "type": "object", @@ -18525,6 +18712,15 @@ } } }, + "TimelineList": { + "description": "TimelineList", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/TimelineComment" + } + } + }, "TopicListResponse": { "description": "TopicListResponse", "schema": { From 6a3611cc3d3b2e401464a855f4630f606f52eb67 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Sun, 2 Jan 2022 00:28:42 +0000 Subject: [PATCH 06/22] [skip ci] Updated licenses and gitignores --- options/gitignore/Jekyll | 3 ++ options/gitignore/Umbraco | 13 +++++++++ options/gitignore/VisualStudioCode | 2 +- options/gitignore/core | 38 +++++++++++++++++++++++++ options/license/App-s2p | 5 ++++ options/license/DL-DE-BY-2.0 | 45 ++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 options/gitignore/core create mode 100644 options/license/App-s2p create mode 100644 options/license/DL-DE-BY-2.0 diff --git a/options/gitignore/Jekyll b/options/gitignore/Jekyll index 2ca868298ced..74a9223a450d 100644 --- a/options/gitignore/Jekyll +++ b/options/gitignore/Jekyll @@ -2,3 +2,6 @@ _site/ .sass-cache/ .jekyll-cache/ .jekyll-metadata +# Ignore folders generated by Bundler +.bundle/ +vendor/ diff --git a/options/gitignore/Umbraco b/options/gitignore/Umbraco index cea9c890c3dd..86b91f300dac 100644 --- a/options/gitignore/Umbraco +++ b/options/gitignore/Umbraco @@ -34,3 +34,16 @@ # Ignore the Models Builder models out of date flag **/ood.flag + +# NEW for version 9 .Net 5 (Core) +#ignore umbraco backoffice assest from wwwroot +**/wwwroot/umbraco/ + +#ignore umbraco data/views/settings +**/umbraco/ + +#include default location for modelsbuilder output +!**/umbraco/models + +#include default location for packages +!**/umbraco/Data/packages \ No newline at end of file diff --git a/options/gitignore/VisualStudioCode b/options/gitignore/VisualStudioCode index 7478c2746eb1..45fce1d71cdb 100644 --- a/options/gitignore/VisualStudioCode +++ b/options/gitignore/VisualStudioCode @@ -1,4 +1,4 @@ -.vscode/ +.vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json diff --git a/options/gitignore/core b/options/gitignore/core new file mode 100644 index 000000000000..c4d93934ad87 --- /dev/null +++ b/options/gitignore/core @@ -0,0 +1,38 @@ +*.swp +*.*~ +project.lock.json +.DS_Store +*.pyc +nupkg/ + +# Visual Studio Code +.vscode + +# Rider +.idea + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +build/ +bld/ +[Bb]in/ +[Oo]bj/ +[Oo]ut/ +msbuild.log +msbuild.err +msbuild.wrn + +# Visual Studio 2015 +.vs/ + diff --git a/options/license/App-s2p b/options/license/App-s2p new file mode 100644 index 000000000000..b19eabf068f1 --- /dev/null +++ b/options/license/App-s2p @@ -0,0 +1,5 @@ +COPYRIGHT and LICENSE + +This program is free and open software. You may use, modify, +distribute, and sell this program (and any modified variants) in any +way you wish, provided you do not restrict others from doing the same. diff --git a/options/license/DL-DE-BY-2.0 b/options/license/DL-DE-BY-2.0 new file mode 100644 index 000000000000..20c3a19c2f52 --- /dev/null +++ b/options/license/DL-DE-BY-2.0 @@ -0,0 +1,45 @@ +DL-DE->BY-2.0 +Datenlizenz Deutschland – Namensnennung – Version 2.0 + +(1) Jede Nutzung ist unter den Bedingungen dieser „Datenlizenz Deutschland – Namensnennung – Version 2.0" zulässig. + +Die bereitgestellten Daten und Metadaten dürfen für die kommerzielle und nicht kommerzielle Nutzung insbesondere + + vervielfältigt, ausgedruckt, präsentiert, verändert, bearbeitet sowie an Dritte übermittelt werden; + mit eigenen Daten und Daten Anderer zusammengeführt und zu selbständigen neuen Datensätzen verbunden werden; + in interne und externe Geschäftsprozesse, Produkte und Anwendungen in öffentlichen und nicht öffentlichen elektronischen Netzwerken eingebunden werden. + +(2) Bei der Nutzung ist sicherzustellen, dass folgende Angaben als Quellenvermerk enthalten sind: + + Bezeichnung des Bereitstellers nach dessen Maßgabe, + der Vermerk „Datenlizenz Deutschland – Namensnennung – Version 2.0" oder „dl-de/by-2-0" mit Verweis auf den Lizenztext unter www.govdata.de/dl-de/by-2-0 sowie + einen Verweis auf den Datensatz (URI). + +Dies gilt nur soweit die datenhaltende Stelle die Angaben 1. bis 3. zum Quellenvermerk bereitstellt. + +(3) Veränderungen, Bearbeitungen, neue Gestaltungen oder sonstige Abwandlungen sind im Quellenvermerk mit dem Hinweis zu versehen, dass die Daten geändert wurden. + + +Data licence Germany – attribution – version 2.0 + +(1) Any use will be permitted provided it fulfils the requirements of this "Data licence Germany – attribution – Version 2.0". + +The data and meta-data provided may, for commercial and non-commercial use, in particular + + be copied, printed, presented, altered, processed and transmitted to third parties; + be merged with own data and with the data of others and be combined to form new and independent datasets; + be integrated in internal and external business processes, products and applications in public and non-public electronic networks. + +(2) The user must ensure that the source note contains the following information: + + the name of the provider, + the annotation "Data licence Germany – attribution – Version 2.0" or "dl-de/by-2-0" referring to the licence text available at www.govdata.de/dl-de/by-2-0, and + a reference to the dataset (URI). + +This applies only if the entity keeping the data provides the pieces of information 1-3 for the source note. + +(3) Changes, editing, new designs or other amendments must be marked as such in the source note. + + + +URL: http://www.govdata.de/dl-de/by-2-0 From 5d4ee44ed3489df04854478b1f5a3d9870beb58e Mon Sep 17 00:00:00 2001 From: singuliere <35190819+singuliere@users.noreply.github.com> Date: Sun, 2 Jan 2022 02:12:13 +0100 Subject: [PATCH 07/22] Add singuliere to MAINTAINERS (#18148) Signed-off-by: singuliere --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index b12877cc0bfb..bbcdde333aeb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -45,3 +45,4 @@ Steven Kriegler (@justusbunsi) Jimmy Praet (@jpraet) Leon Hofmeister (@delvh) Gusted (@singuliere) From 88da7a7174f9c1568cc2d8d084d6b05a8d268690 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 2 Jan 2022 02:38:07 +0000 Subject: [PATCH 08/22] Use correct user when determining max repo limits for error messages (#18153) - Use the correct user(`owner` instead of `ctx.User`) to get the maxCreationLimit. --- routers/web/repo/migrate.go | 2 +- routers/web/repo/repo.go | 2 +- routers/web/repo/setting.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index 9b1265383908..1bbb192b248a 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -82,7 +82,7 @@ func handleMigrateError(ctx *context.Context, owner *user_model.User, err error, ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form) case repo_model.IsErrReachLimitOfRepo(err): var msg string - maxCreationLimit := ctx.User.MaxCreationLimit() + maxCreationLimit := owner.MaxCreationLimit() if maxCreationLimit == 1 { msg = ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit) } else { diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 0d8c6d374f98..8e0998225aa6 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -163,7 +163,7 @@ func handleCreateError(ctx *context.Context, owner *user_model.User, err error, switch { case repo_model.IsErrReachLimitOfRepo(err): var msg string - maxCreationLimit := ctx.User.MaxCreationLimit() + maxCreationLimit := owner.MaxCreationLimit() if maxCreationLimit == 1 { msg = ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit) } else { diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 5457e651d38f..57a195ee625d 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -609,7 +609,7 @@ func SettingsPost(ctx *context.Context) { } if !ctx.Repo.Owner.CanCreateRepo() { - maxCreationLimit := ctx.User.MaxCreationLimit() + maxCreationLimit := ctx.Repo.Owner.MaxCreationLimit() if maxCreationLimit == 1 { ctx.Flash.Error(ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit)) } else { From e61b390d545919244141b699b28e3fbc42adc66f Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 2 Jan 2022 11:33:57 +0800 Subject: [PATCH 09/22] Unify and simplify TrN for i18n (#18141) Refer: https://github.com/go-gitea/gitea/pull/18135#issuecomment-1003246099 Now we have a unique and simple `TrN`, and make the fix of PR #18135 also use the better `TrN` logic. --- modules/csv/csv_test.go | 36 ++++++---- modules/templates/helper.go | 64 ------------------ modules/test/context_tests.go | 4 ++ modules/translation/translation.go | 65 +++++++++++++++++++ routers/web/repo/migrate.go | 7 +- routers/web/repo/repo.go | 7 +- routers/web/repo/setting.go | 7 +- services/mailer/mail.go | 5 -- services/mailer/mail_release.go | 1 - services/mailer/mail_repo.go | 1 - templates/mail/issue/default.tmpl | 2 +- templates/repo/activity.tmpl | 55 ++++++++++------ templates/repo/blame.tmpl | 2 +- templates/repo/create.tmpl | 2 +- .../repo/issue/view_content/comments.tmpl | 6 +- templates/repo/issue/view_content/pull.tmpl | 4 +- templates/repo/sub_menu.tmpl | 6 +- templates/repo/view_file.tmpl | 2 +- templates/shared/issuelist.tmpl | 8 +-- 19 files changed, 148 insertions(+), 136 deletions(-) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index d72c3e3a7329..1612a66953af 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/csv" "io" + "strconv" "strings" "testing" @@ -21,14 +22,21 @@ func TestCreateReader(t *testing.T) { assert.Equal(t, ',', rd.Comma) } -//nolint +func decodeSlashes(t *testing.T, s string) string { + s = strings.ReplaceAll(s, "\n", "\\n") + s = strings.ReplaceAll(s, "\"", "\\\"") + decoded, err := strconv.Unquote(`"` + s + `"`) + assert.NoError(t, err, "unable to decode string") + return decoded +} + func TestCreateReaderAndDetermineDelimiter(t *testing.T) { var cases = []struct { csv string expectedRows [][]string expectedDelimiter rune }{ - // case 0 - semicolon delmited + // case 0 - semicolon delimited { csv: `a;b;c 1;2;3 @@ -47,11 +55,11 @@ a, b c e f g h i j l -m n, +m n,\t p q r u v w x -y +y\t\t `, expectedRows: [][]string{ {"col1", "col2", "col3"}, @@ -74,7 +82,7 @@ y a, b, c d,e,f ,h, i -j, , +j, ,\x20 , , `, expectedRows: [][]string{ {"col1", "col2", "col3"}, @@ -89,7 +97,7 @@ j, , } for n, c := range cases { - rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.csv)) + rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(decodeSlashes(t, c.csv))) assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) assert.EqualValues(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) rows, err := rd.ReadAll() @@ -222,7 +230,7 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, } for n, c := range cases { - delimiter := determineDelimiter(&markup.RenderContext{Filename: c.filename}, []byte(c.csv)) + delimiter := determineDelimiter(&markup.RenderContext{Filename: c.filename}, []byte(decodeSlashes(t, c.csv))) assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -287,7 +295,7 @@ abc | |123 } for n, c := range cases { - modifiedText := removeQuotedString(c.text) + modifiedText := removeQuotedString(decodeSlashes(t, c.text)) assert.EqualValues(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) } } @@ -353,7 +361,7 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, quoted, text," a 2 "some, -quoted, +quoted,\t text," b 3 "some, quoted, @@ -442,7 +450,7 @@ jkl`, } for n, c := range cases { - delimiter := guessDelimiter([]byte(c.csv)) + delimiter := guessDelimiter([]byte(decodeSlashes(t, c.csv))) assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -459,7 +467,7 @@ func TestGuessFromBeforeAfterQuotes(t *testing.T) { quoted, text," a 2 "some, -quoted, +quoted,\t text," b 3 "some, quoted, @@ -534,7 +542,7 @@ a|"he said, ""here I am"""`, } for n, c := range cases { - delimiter := guessFromBeforeAfterQuotes([]byte(c.csv)) + delimiter := guessFromBeforeAfterQuotes([]byte(decodeSlashes(t, c.csv))) assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } @@ -549,6 +557,10 @@ func (l mockLocale) Tr(s string, _ ...interface{}) string { return s } +func (l mockLocale) TrN(_cnt interface{}, key1, _keyN string, _args ...interface{}) string { + return key1 +} + func TestFormatError(t *testing.T) { var cases = []struct { err error diff --git a/modules/templates/helper.go b/modules/templates/helper.go index a05c0c1a95ff..fc07b49c7192 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -239,7 +239,6 @@ func NewFuncMap() []template.FuncMap { "DisableImportLocal": func() bool { return !setting.ImportLocalPaths }, - "TrN": TrN, "Dict": func(values ...interface{}) (map[string]interface{}, error) { if len(values)%2 != 0 { return nil, errors.New("invalid dict call") @@ -857,69 +856,6 @@ func DiffLineTypeToStr(diffType int) string { return "same" } -// Language specific rules for translating plural texts -var trNLangRules = map[string]func(int64) int{ - "en-US": func(cnt int64) int { - if cnt == 1 { - return 0 - } - return 1 - }, - "lv-LV": func(cnt int64) int { - if cnt%10 == 1 && cnt%100 != 11 { - return 0 - } - return 1 - }, - "ru-RU": func(cnt int64) int { - if cnt%10 == 1 && cnt%100 != 11 { - return 0 - } - return 1 - }, - "zh-CN": func(cnt int64) int { - return 0 - }, - "zh-HK": func(cnt int64) int { - return 0 - }, - "zh-TW": func(cnt int64) int { - return 0 - }, - "fr-FR": func(cnt int64) int { - if cnt > -2 && cnt < 2 { - return 0 - } - return 1 - }, -} - -// TrN returns key to be used for plural text translation -func TrN(lang string, cnt interface{}, key1, keyN string) string { - var c int64 - if t, ok := cnt.(int); ok { - c = int64(t) - } else if t, ok := cnt.(int16); ok { - c = int64(t) - } else if t, ok := cnt.(int32); ok { - c = int64(t) - } else if t, ok := cnt.(int64); ok { - c = t - } else { - return keyN - } - - ruleFunc, ok := trNLangRules[lang] - if !ok { - ruleFunc = trNLangRules["en-US"] - } - - if ruleFunc(c) == 0 { - return key1 - } - return keyN -} - // MigrationIcon returns a SVG name matching the service an issue/comment was migrated from func MigrationIcon(hostname string) string { switch hostname { diff --git a/modules/test/context_tests.go b/modules/test/context_tests.go index 1f893122f3f7..62ec21f6fe17 100644 --- a/modules/test/context_tests.go +++ b/modules/test/context_tests.go @@ -103,6 +103,10 @@ func (l mockLocale) Tr(s string, _ ...interface{}) string { return s } +func (l mockLocale) TrN(_cnt interface{}, key1, _keyN string, _args ...interface{}) string { + return key1 +} + type mockResponseWriter struct { httptest.ResponseRecorder size int diff --git a/modules/translation/translation.go b/modules/translation/translation.go index 77cc9ac7f584..af1e5d25df53 100644 --- a/modules/translation/translation.go +++ b/modules/translation/translation.go @@ -17,6 +17,7 @@ import ( type Locale interface { Language() string Tr(string, ...interface{}) string + TrN(cnt interface{}, key1, keyN string, args ...interface{}) string } // LangType represents a lang type @@ -99,3 +100,67 @@ func (l *locale) Language() string { func (l *locale) Tr(format string, args ...interface{}) string { return i18n.Tr(l.Lang, format, args...) } + +// Language specific rules for translating plural texts +var trNLangRules = map[string]func(int64) int{ + // the default rule is "en-US" if a language isn't listed here + "en-US": func(cnt int64) int { + if cnt == 1 { + return 0 + } + return 1 + }, + "lv-LV": func(cnt int64) int { + if cnt%10 == 1 && cnt%100 != 11 { + return 0 + } + return 1 + }, + "ru-RU": func(cnt int64) int { + if cnt%10 == 1 && cnt%100 != 11 { + return 0 + } + return 1 + }, + "zh-CN": func(cnt int64) int { + return 0 + }, + "zh-HK": func(cnt int64) int { + return 0 + }, + "zh-TW": func(cnt int64) int { + return 0 + }, + "fr-FR": func(cnt int64) int { + if cnt > -2 && cnt < 2 { + return 0 + } + return 1 + }, +} + +// TrN returns translated message for plural text translation +func (l *locale) TrN(cnt interface{}, key1, keyN string, args ...interface{}) string { + var c int64 + if t, ok := cnt.(int); ok { + c = int64(t) + } else if t, ok := cnt.(int16); ok { + c = int64(t) + } else if t, ok := cnt.(int32); ok { + c = int64(t) + } else if t, ok := cnt.(int64); ok { + c = t + } else { + return l.Tr(keyN, args...) + } + + ruleFunc, ok := trNLangRules[l.Lang] + if !ok { + ruleFunc = trNLangRules["en-US"] + } + + if ruleFunc(c) == 0 { + return l.Tr(key1, args...) + } + return l.Tr(keyN, args...) +} diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index 1bbb192b248a..23e5b21b4806 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -81,13 +81,8 @@ func handleMigrateError(ctx *context.Context, owner *user_model.User, err error, case migrations.IsTwoFactorAuthError(err): ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form) case repo_model.IsErrReachLimitOfRepo(err): - var msg string maxCreationLimit := owner.MaxCreationLimit() - if maxCreationLimit == 1 { - msg = ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit) - } else { - msg = ctx.Tr("repo.form.reach_limit_of_creation_n", maxCreationLimit) - } + msg := ctx.TrN(maxCreationLimit, "repo.form.reach_limit_of_creation_1", "repo.form.reach_limit_of_creation_n", maxCreationLimit) ctx.RenderWithErr(msg, tpl, form) case repo_model.IsErrRepoAlreadyExist(err): ctx.Data["Err_RepoName"] = true diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 8e0998225aa6..6bd16ff2b86a 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -162,13 +162,8 @@ func Create(ctx *context.Context) { func handleCreateError(ctx *context.Context, owner *user_model.User, err error, name string, tpl base.TplName, form interface{}) { switch { case repo_model.IsErrReachLimitOfRepo(err): - var msg string maxCreationLimit := owner.MaxCreationLimit() - if maxCreationLimit == 1 { - msg = ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit) - } else { - msg = ctx.Tr("repo.form.reach_limit_of_creation_n", maxCreationLimit) - } + msg := ctx.TrN(maxCreationLimit, "repo.form.reach_limit_of_creation_1", "repo.form.reach_limit_of_creation_n", maxCreationLimit) ctx.RenderWithErr(msg, tpl, form) case repo_model.IsErrRepoAlreadyExist(err): ctx.Data["Err_RepoName"] = true diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 57a195ee625d..7ff83cc3d76b 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -610,11 +610,8 @@ func SettingsPost(ctx *context.Context) { if !ctx.Repo.Owner.CanCreateRepo() { maxCreationLimit := ctx.Repo.Owner.MaxCreationLimit() - if maxCreationLimit == 1 { - ctx.Flash.Error(ctx.Tr("repo.form.reach_limit_of_creation_1", maxCreationLimit)) - } else { - ctx.Flash.Error(ctx.Tr("repo.form.reach_limit_of_creation_n", maxCreationLimit)) - } + msg := ctx.TrN(maxCreationLimit, "repo.form.reach_limit_of_creation_1", "repo.form.reach_limit_of_creation_n", maxCreationLimit) + ctx.Flash.Error(msg) ctx.Redirect(repo.Link() + "/settings") return } diff --git a/services/mailer/mail.go b/services/mailer/mail.go index 0a5573707396..20552be58478 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -78,7 +78,6 @@ func sendUserMail(language string, u *user_model.User, tpl base.TplName, code, s // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var content bytes.Buffer @@ -129,7 +128,6 @@ func SendActivateEmailMail(u *user_model.User, email *user_model.EmailAddress) { // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var content bytes.Buffer @@ -160,7 +158,6 @@ func SendRegisterNotifyMail(u *user_model.User) { // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var content bytes.Buffer @@ -194,7 +191,6 @@ func SendCollaboratorMail(u, doer *user_model.User, repo *repo_model.Repository) // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var content bytes.Buffer @@ -278,7 +274,6 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var mailSubject bytes.Buffer diff --git a/services/mailer/mail_release.go b/services/mailer/mail_release.go index 02aa0f312e36..ee4c6f3a59cd 100644 --- a/services/mailer/mail_release.go +++ b/services/mailer/mail_release.go @@ -76,7 +76,6 @@ func mailNewRelease(lang string, tos []string, rel *models.Release) { // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } var mailBody bytes.Buffer diff --git a/services/mailer/mail_repo.go b/services/mailer/mail_repo.go index 51b16aa7e270..a5343f81286e 100644 --- a/services/mailer/mail_repo.go +++ b/services/mailer/mail_repo.go @@ -71,7 +71,6 @@ func sendRepoTransferNotifyMailPerLang(lang string, newOwner, doer *user_model.U // helper "i18n": locale, "Str2html": templates.Str2html, - "TrN": templates.TrN, } if err := bodyTemplates.ExecuteTemplate(&content, string(mailRepoTransferNotify), data); err != nil { diff --git a/templates/mail/issue/default.tmpl b/templates/mail/issue/default.tmpl index e01ec667eedc..aa91861d11ab 100644 --- a/templates/mail/issue/default.tmpl +++ b/templates/mail/issue/default.tmpl @@ -30,7 +30,7 @@ {{.i18n.Tr "mail.issue.action.force_push" .Doer.Name .Comment.Issue.PullRequest.HeadBranch $oldCommitLink $newCommitLink | Str2html}} {{else}} - {{.i18n.Tr (TrN .i18n.Lang (len .Comment.Commits) "mail.issue.action.push_1" "mail.issue.action.push_n") .Doer.Name .Comment.Issue.PullRequest.HeadBranch (len .Comment.Commits) | Str2html}} + {{.i18n.TrN (len .Comment.Commits) "mail.issue.action.push_1" "mail.issue.action.push_n" .Doer.Name .Comment.Issue.PullRequest.HeadBranch (len .Comment.Commits) | Str2html}} {{end}}

{{end}} diff --git a/templates/repo/activity.tmpl b/templates/repo/activity.tmpl index c67925e4233c..3086ca8e8d3c 100644 --- a/templates/repo/activity.tmpl +++ b/templates/repo/activity.tmpl @@ -41,7 +41,7 @@ {{end}} - {{.i18n.Tr (TrN .i18n.Lang .Activity.ActivePRCount "repo.activity.active_prs_count_1" "repo.activity.active_prs_count_n") .Activity.ActivePRCount | Safe }} + {{.i18n.TrN .Activity.ActivePRCount "repo.activity.active_prs_count_1" "repo.activity.active_prs_count_n" .Activity.ActivePRCount | Safe }} {{end}} {{if .Permission.CanRead $.UnitTypeIssues}} @@ -56,7 +56,7 @@ {{end}} - {{.i18n.Tr (TrN .i18n.Lang .Activity.ActiveIssueCount "repo.activity.active_issues_count_1" "repo.activity.active_issues_count_n") .Activity.ActiveIssueCount | Safe }} + {{.i18n.TrN .Activity.ActiveIssueCount "repo.activity.active_issues_count_1" "repo.activity.active_issues_count_n" .Activity.ActiveIssueCount | Safe }} {{end}} @@ -64,21 +64,21 @@ {{if .Permission.CanRead $.UnitTypePullRequests}} {{svg "octicon-git-pull-request"}} {{.Activity.MergedPRCount}}
- {{.i18n.Tr (TrN .i18n.Lang .Activity.MergedPRCount "repo.activity.merged_prs_count_1" "repo.activity.merged_prs_count_n") }} + {{.i18n.TrN .Activity.MergedPRCount "repo.activity.merged_prs_count_1" "repo.activity.merged_prs_count_n"}} {{svg "octicon-git-branch"}} {{.Activity.OpenedPRCount}}
- {{.i18n.Tr (TrN .i18n.Lang .Activity.OpenedPRCount "repo.activity.opened_prs_count_1" "repo.activity.opened_prs_count_n") }} + {{.i18n.TrN .Activity.OpenedPRCount "repo.activity.opened_prs_count_1" "repo.activity.opened_prs_count_n"}} {{end}} {{if .Permission.CanRead $.UnitTypeIssues}} {{svg "octicon-issue-closed"}} {{.Activity.ClosedIssueCount}}
- {{.i18n.Tr (TrN .i18n.Lang .Activity.ClosedIssueCount "repo.activity.closed_issues_count_1" "repo.activity.closed_issues_count_n") }} + {{.i18n.TrN .Activity.ClosedIssueCount "repo.activity.closed_issues_count_1" "repo.activity.closed_issues_count_n"}} {{svg "octicon-issue-opened"}} {{.Activity.OpenedIssueCount}}
- {{.i18n.Tr (TrN .i18n.Lang .Activity.OpenedIssueCount "repo.activity.new_issues_count_1" "repo.activity.new_issues_count_n") }} + {{.i18n.TrN .Activity.OpenedIssueCount "repo.activity.new_issues_count_1" "repo.activity.new_issues_count_n"}} {{end}} @@ -94,19 +94,19 @@
{{.i18n.Tr "repo.activity.git_stats_exclude_merges" }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.AuthorCount "repo.activity.git_stats_author_1" "repo.activity.git_stats_author_n") .Activity.Code.AuthorCount }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.AuthorCount "repo.activity.git_stats_pushed_1" "repo.activity.git_stats_pushed_n") }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.CommitCount "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n") .Activity.Code.CommitCount }} + {{.i18n.TrN .Activity.Code.AuthorCount "repo.activity.git_stats_author_1" "repo.activity.git_stats_author_n" .Activity.Code.AuthorCount}} + {{.i18n.TrN .Activity.Code.AuthorCount "repo.activity.git_stats_pushed_1" "repo.activity.git_stats_pushed_n"}} + {{.i18n.TrN .Activity.Code.CommitCount "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n" .Activity.Code.CommitCount}} {{.i18n.Tr "repo.activity.git_stats_push_to_branch" .Repository.DefaultBranch }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.CommitCountInAllBranches "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n") .Activity.Code.CommitCountInAllBranches }} + {{.i18n.TrN .Activity.Code.CommitCountInAllBranches "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n" .Activity.Code.CommitCountInAllBranches}} {{.i18n.Tr "repo.activity.git_stats_push_to_all_branches" }} {{.i18n.Tr "repo.activity.git_stats_on_default_branch" .Repository.DefaultBranch }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.ChangedFiles "repo.activity.git_stats_file_1" "repo.activity.git_stats_file_n") .Activity.Code.ChangedFiles }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.ChangedFiles "repo.activity.git_stats_files_changed_1" "repo.activity.git_stats_files_changed_n") }} + {{.i18n.TrN .Activity.Code.ChangedFiles "repo.activity.git_stats_file_1" "repo.activity.git_stats_file_n" .Activity.Code.ChangedFiles}} + {{.i18n.TrN .Activity.Code.ChangedFiles "repo.activity.git_stats_files_changed_1" "repo.activity.git_stats_files_changed_n"}} {{.i18n.Tr "repo.activity.git_stats_additions" }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.Additions "repo.activity.git_stats_addition_1" "repo.activity.git_stats_addition_n") .Activity.Code.Additions }} + {{.i18n.TrN .Activity.Code.Additions "repo.activity.git_stats_addition_1" "repo.activity.git_stats_addition_n" .Activity.Code.Additions}} {{.i18n.Tr "repo.activity.git_stats_and_deletions" }} - {{.i18n.Tr (TrN .i18n.Lang .Activity.Code.Deletions "repo.activity.git_stats_deletion_1" "repo.activity.git_stats_deletion_n") .Activity.Code.Deletions }}. + {{.i18n.TrN .Activity.Code.Deletions "repo.activity.git_stats_deletion_1" "repo.activity.git_stats_deletion_n" .Activity.Code.Deletions}}.
@@ -118,7 +118,10 @@ {{if gt .Activity.PublishedReleaseCount 0}}

{{svg "octicon-tag"}} - {{.i18n.Tr "repo.activity.title.releases_published_by" (.i18n.Tr (TrN .i18n.Lang .Activity.PublishedReleaseCount "repo.activity.title.releases_1" "repo.activity.title.releases_n") .Activity.PublishedReleaseCount) (.i18n.Tr (TrN .i18n.Lang .Activity.PublishedReleaseAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n") .Activity.PublishedReleaseAuthorCount) }} + {{.i18n.Tr "repo.activity.title.releases_published_by" + (.i18n.TrN .Activity.PublishedReleaseCount "repo.activity.title.releases_1" "repo.activity.title.releases_n" .Activity.PublishedReleaseCount) + (.i18n.TrN .Activity.PublishedReleaseAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n" .Activity.PublishedReleaseAuthorCount) + }}

{{range .Activity.PublishedReleases}} @@ -137,7 +140,10 @@ {{if gt .Activity.MergedPRCount 0}}

{{svg "octicon-git-pull-request"}} - {{.i18n.Tr "repo.activity.title.prs_merged_by" (.i18n.Tr (TrN .i18n.Lang .Activity.MergedPRCount "repo.activity.title.prs_1" "repo.activity.title.prs_n") .Activity.MergedPRCount) (.i18n.Tr (TrN .i18n.Lang .Activity.MergedPRAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n") .Activity.MergedPRAuthorCount) }} + {{.i18n.Tr "repo.activity.title.prs_merged_by" + (.i18n.TrN .Activity.MergedPRCount "repo.activity.title.prs_1" "repo.activity.title.prs_n" .Activity.MergedPRCount) + (.i18n.TrN .Activity.MergedPRAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n" .Activity.MergedPRAuthorCount) + }}

{{range .Activity.MergedPRs}} @@ -153,7 +159,10 @@ {{if gt .Activity.OpenedPRCount 0}}

{{svg "octicon-git-branch"}} - {{.i18n.Tr "repo.activity.title.prs_opened_by" (.i18n.Tr (TrN .i18n.Lang .Activity.OpenedPRCount "repo.activity.title.prs_1" "repo.activity.title.prs_n") .Activity.OpenedPRCount) (.i18n.Tr (TrN .i18n.Lang .Activity.OpenedPRAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n") .Activity.OpenedPRAuthorCount) }} + {{.i18n.Tr "repo.activity.title.prs_opened_by" + (.i18n.TrN .Activity.OpenedPRCount "repo.activity.title.prs_1" "repo.activity.title.prs_n" .Activity.OpenedPRCount) + (.i18n.TrN .Activity.OpenedPRAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n" .Activity.OpenedPRAuthorCount) + }}

{{range .Activity.OpenedPRs}} @@ -169,7 +178,10 @@ {{if gt .Activity.ClosedIssueCount 0}}

{{svg "octicon-issue-closed"}} - {{.i18n.Tr "repo.activity.title.issues_closed_from" (.i18n.Tr (TrN .i18n.Lang .Activity.ClosedIssueCount "repo.activity.title.issues_1" "repo.activity.title.issues_n") .Activity.ClosedIssueCount) (.i18n.Tr (TrN .i18n.Lang .Activity.ClosedIssueAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n") .Activity.ClosedIssueAuthorCount) }} + {{.i18n.Tr "repo.activity.title.issues_closed_from" + (.i18n.TrN .Activity.ClosedIssueCount "repo.activity.title.issues_1" "repo.activity.title.issues_n" .Activity.ClosedIssueCount) + (.i18n.TrN .Activity.ClosedIssueAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n" .Activity.ClosedIssueAuthorCount) + }}

{{range .Activity.ClosedIssues}} @@ -185,7 +197,10 @@ {{if gt .Activity.OpenedIssueCount 0}}

{{svg "octicon-issue-opened"}} - {{.i18n.Tr "repo.activity.title.issues_created_by" (.i18n.Tr (TrN .i18n.Lang .Activity.OpenedIssueCount "repo.activity.title.issues_1" "repo.activity.title.issues_n") .Activity.OpenedIssueCount) (.i18n.Tr (TrN .i18n.Lang .Activity.OpenedIssueAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n") .Activity.OpenedIssueAuthorCount) }} + {{.i18n.Tr "repo.activity.title.issues_created_by" + (.i18n.TrN .Activity.OpenedIssueCount "repo.activity.title.issues_1" "repo.activity.title.issues_n" .Activity.OpenedIssueCount) + (.i18n.TrN .Activity.OpenedIssueAuthorCount "repo.activity.title.user_1" "repo.activity.title.user_n" .Activity.OpenedIssueAuthorCount) + }}

{{range .Activity.OpenedIssues}} @@ -201,7 +216,7 @@ {{if gt .Activity.UnresolvedIssueCount 0}}

{{svg "octicon-comment-discussion"}} - {{.i18n.Tr (TrN .i18n.Lang .Activity.UnresolvedIssueCount "repo.activity.title.unresolved_conv_1" "repo.activity.title.unresolved_conv_n") .Activity.UnresolvedIssueCount }} + {{.i18n.TrN .Activity.UnresolvedIssueCount "repo.activity.title.unresolved_conv_1" "repo.activity.title.unresolved_conv_n" .Activity.UnresolvedIssueCount}}

{{.i18n.Tr "repo.activity.unresolved_conv_desc"}} diff --git a/templates/repo/blame.tmpl b/templates/repo/blame.tmpl index 0b37f41c28c4..cdd31c0eba7a 100644 --- a/templates/repo/blame.tmpl +++ b/templates/repo/blame.tmpl @@ -3,7 +3,7 @@
- {{.NumLines}} {{.i18n.Tr (TrN .i18n.Lang .NumLines "repo.line" "repo.lines") }} + {{.NumLines}} {{.i18n.TrN .NumLines "repo.line" "repo.lines"}}
{{FileSize .FileSize}}
diff --git a/templates/repo/create.tmpl b/templates/repo/create.tmpl index c7aaf5435133..c250b0d4a988 100644 --- a/templates/repo/create.tmpl +++ b/templates/repo/create.tmpl @@ -16,7 +16,7 @@ {{if not .CanCreateRepo}}
-

{{.i18n.Tr (TrN .i18n.Lang .MaxCreationLimit "repo.form.reach_limit_of_creation_1" "repo.form.reach_limit_of_creation_n") .MaxCreationLimit}}

+

{{.i18n.TrN .MaxCreationLimit "repo.form.reach_limit_of_creation_1" "repo.form.reach_limit_of_creation_n" .MaxCreationLimit}}

{{end}}
diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index c229028540d7..026e1de0fd1a 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -187,9 +187,9 @@ {{.Poster.GetDisplayName}} {{if and .AddedLabels (not .RemovedLabels)}} - {{$.i18n.Tr (TrN $.i18n.Lang (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels") (RenderLabels .AddedLabels) $createdStr | Safe}} + {{$.i18n.TrN (len .AddedLabels) "repo.issues.add_label" "repo.issues.add_labels" (RenderLabels .AddedLabels) $createdStr | Safe}} {{else if and (not .AddedLabels) .RemovedLabels}} - {{$.i18n.Tr (TrN $.i18n.Lang (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels") (RenderLabels .RemovedLabels) $createdStr | Safe}} + {{$.i18n.TrN (len .RemovedLabels) "repo.issues.remove_label" "repo.issues.remove_labels" (RenderLabels .RemovedLabels) $createdStr | Safe}} {{else}} {{$.i18n.Tr "repo.issues.add_remove_labels" (RenderLabels .AddedLabels) (RenderLabels .RemovedLabels) $createdStr | Safe}} {{end}} @@ -716,7 +716,7 @@ {{ if .IsForcePush }} {{$.i18n.Tr "repo.issues.force_push_codes" ($.Issue.PullRequest.HeadBranch|Escape) (ShortSha .OldCommit) (($.Issue.Repo.CommitLink .OldCommit)|Escape) (ShortSha .NewCommit) (($.Issue.Repo.CommitLink .NewCommit)|Escape) $createdStr | Safe}} {{else}} - {{$.i18n.Tr (TrN $.i18n.Lang (len .Commits) "repo.issues.push_commit_1" "repo.issues.push_commits_n") (len .Commits) $createdStr | Safe}} + {{$.i18n.TrN (len .Commits) "repo.issues.push_commit_1" "repo.issues.push_commits_n" (len .Commits) $createdStr | Safe}} {{end}}
diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index bb80bba7c81a..85076027d1cf 100644 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -224,7 +224,7 @@ {{else if .IsBlockedByChangedProtectedFiles}}
{{svg "octicon-x" 16}} - {{$.i18n.Tr (TrN $.i18n.Lang $.ChangedProtectedFilesNum "repo.pulls.blocked_by_changed_protected_files_1" "repo.pulls.blocked_by_changed_protected_files_n") | Safe }} + {{$.i18n.TrN $.ChangedProtectedFilesNum "repo.pulls.blocked_by_changed_protected_files_1" "repo.pulls.blocked_by_changed_protected_files_n" | Safe }}
{{range .ChangedProtectedFiles}}
{{.}}
@@ -555,7 +555,7 @@ {{else if .IsBlockedByChangedProtectedFiles}}
{{svg "octicon-x" 16}} - {{$.i18n.Tr (TrN $.i18n.Lang $.ChangedProtectedFilesNum "repo.pulls.blocked_by_changed_protected_files_1" "repo.pulls.blocked_by_changed_protected_files_n") | Safe }} + {{$.i18n.TrN $.ChangedProtectedFilesNum "repo.pulls.blocked_by_changed_protected_files_1" "repo.pulls.blocked_by_changed_protected_files_n" | Safe }}
{{range .ChangedProtectedFiles}}
{{.}}
diff --git a/templates/repo/sub_menu.tmpl b/templates/repo/sub_menu.tmpl index 5ba88f735660..57bef2e40f2b 100644 --- a/templates/repo/sub_menu.tmpl +++ b/templates/repo/sub_menu.tmpl @@ -3,14 +3,14 @@
{{if and (.Permission.CanRead $.UnitTypeCode) (not .IsEmptyRepo)}}
- {{svg "octicon-history"}} {{.CommitsCount}} {{.i18n.Tr (TrN .i18n.Lang .CommitsCount "repo.commit" "repo.commits") }} + {{svg "octicon-history"}} {{.CommitsCount}} {{.i18n.TrN .CommitsCount "repo.commit" "repo.commits"}}
- {{svg "octicon-git-branch"}} {{.BranchesCount}} {{.i18n.Tr (TrN .i18n.Lang .BranchesCount "repo.branch" "repo.branches") }} + {{svg "octicon-git-branch"}} {{.BranchesCount}} {{.i18n.TrN .BranchesCount "repo.branch" "repo.branches"}}
{{if $.Permission.CanRead $.UnitTypeCode}}
- {{svg "octicon-tag"}} {{.NumTags}} {{.i18n.Tr (TrN .i18n.Lang .NumTags "repo.tag" "repo.tags") }} + {{svg "octicon-tag"}} {{.NumTags}} {{.i18n.TrN .NumTags "repo.tag" "repo.tags"}}
{{end}}
diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 6698e6bbe5ab..6bd54cc8e53a 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -13,7 +13,7 @@ {{end}} {{if .NumLinesSet}}
- {{.NumLines}} {{.i18n.Tr (TrN .i18n.Lang .NumLines "repo.line" "repo.lines") }} + {{.NumLines}} {{.i18n.TrN .NumLines "repo.line" "repo.lines"}}
{{end}} {{if .FileSize}} diff --git a/templates/shared/issuelist.tmpl b/templates/shared/issuelist.tmpl index 0b68a03414b1..3050107123cb 100644 --- a/templates/shared/issuelist.tmpl +++ b/templates/shared/issuelist.tmpl @@ -91,25 +91,25 @@ {{if gt $approveOfficial 0}} {{svg "octicon-check" 14 "mr-1"}} - {{$.i18n.Tr (TrN $.i18n.Lang $approveOfficial "repo.pulls.approve_count_1" "repo.pulls.approve_count_n") $approveOfficial}} + {{$.i18n.TrN $approveOfficial "repo.pulls.approve_count_1" "repo.pulls.approve_count_n" $approveOfficial}} {{end}} {{if gt $rejectOfficial 0}} {{svg "octicon-diff" 14 "mr-2"}} - {{$.i18n.Tr (TrN $.i18n.Lang $rejectOfficial "repo.pulls.reject_count_1" "repo.pulls.reject_count_n") $rejectOfficial}} + {{$.i18n.TrN $rejectOfficial "repo.pulls.reject_count_1" "repo.pulls.reject_count_n" $rejectOfficial}} {{end}} {{if gt $waitingOfficial 0}} {{svg "octicon-eye" 14 "mr-2"}} - {{$.i18n.Tr (TrN $.i18n.Lang $waitingOfficial "repo.pulls.waiting_count_1" "repo.pulls.waiting_count_n") $waitingOfficial}} + {{$.i18n.TrN $waitingOfficial "repo.pulls.waiting_count_1" "repo.pulls.waiting_count_n" $waitingOfficial}} {{end}} {{if and (not .PullRequest.HasMerged) (gt (len .PullRequest.ConflictedFiles) 0)}} {{svg "octicon-x" 14}} - {{$.i18n.Tr (TrN $.i18n.Lang (len .PullRequest.ConflictedFiles) "repo.pulls.num_conflicting_files_1" "repo.pulls.num_conflicting_files_n") (len .PullRequest.ConflictedFiles)}} + {{$.i18n.TrN (len .PullRequest.ConflictedFiles) "repo.pulls.num_conflicting_files_1" "repo.pulls.num_conflicting_files_n" (len .PullRequest.ConflictedFiles)}} {{end}} {{end}} From de8e3948a5e38f7eaf82d3c0cfd10e995bf68e92 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 2 Jan 2022 21:12:35 +0800 Subject: [PATCH 10/22] Refactor auth package (#17962) --- cmd/admin.go | 18 +- cmd/admin_auth_ldap.go | 76 +- cmd/admin_auth_ldap_test.go | 298 +-- integrations/api_oauth2_apps_test.go | 28 +- models/asymkey/ssh_key.go | 32 +- models/asymkey/ssh_key_principals.go | 4 +- models/{login => auth}/main_test.go | 2 +- .../oauth2_application.go => auth/oauth2.go} | 54 +- .../oauth2_test.go} | 2 +- models/{login => auth}/session.go | 2 +- models/{login => auth}/source.go | 44 +- models/{login => auth}/source_test.go | 8 +- models/{login => auth}/twofactor.go | 2 +- models/{login => auth}/u2f.go | 2 +- models/{login => auth}/u2f_test.go | 2 +- models/db/convert.go | 16 + models/login/oauth2.go | 70 - models/repo/repo_unit.go | 3 +- models/statistic.go | 6 +- models/token.go | 6 +- models/user/list.go | 6 +- models/user/user.go | 10 +- models/user/user_test.go | 4 +- modules/context/api.go | 14 +- modules/context/auth.go | 6 +- modules/convert/convert.go | 6 +- modules/metrics/collector.go | 2 +- modules/session/db.go | 16 +- routers/api/v1/admin/user.go | 16 +- routers/api/v1/user/app.go | 18 +- routers/web/admin/auths.go | 88 +- routers/web/admin/users.go | 44 +- routers/web/auth/2fa.go | 166 ++ routers/web/auth/auth.go | 795 +++++++ routers/web/auth/linkaccount.go | 300 +++ routers/web/auth/main_test.go | 16 + routers/web/{user => auth}/oauth.go | 400 +++- routers/web/{user => auth}/oauth_test.go | 10 +- .../{user/auth_openid.go => auth/openid.go} | 2 +- routers/web/auth/password.go | 346 +++ routers/web/auth/u2f.go | 136 ++ routers/web/events/events.go | 4 +- routers/web/home.go | 3 +- routers/web/repo/http.go | 6 +- routers/web/user/auth.go | 2006 ----------------- routers/web/user/setting/applications.go | 6 +- routers/web/user/setting/keys.go | 1 - routers/web/user/setting/oauth2.go | 18 +- .../{security_twofa.go => security/2fa.go} | 30 +- .../openid.go} | 2 +- .../user/setting/{ => security}/security.go | 26 +- .../{security_u2f.go => security/u2f.go} | 14 +- routers/web/web.go | 164 +- services/asymkey/sign.go | 18 +- services/asymkey/ssh_key_test.go | 4 +- services/auth/oauth2.go | 6 +- services/auth/signin.go | 10 +- services/auth/{login_source.go => source.go} | 14 +- .../auth/source/db/assert_interface_test.go | 4 +- services/auth/source/db/source.go | 6 +- .../auth/source/ldap/assert_interface_test.go | 14 +- services/auth/source/ldap/source.go | 16 +- .../auth/source/ldap/source_authenticate.go | 12 +- services/auth/source/ldap/source_sync.go | 40 +- .../source/oauth2/assert_interface_test.go | 8 +- services/auth/source/oauth2/init.go | 14 +- services/auth/source/oauth2/providers.go | 14 +- services/auth/source/oauth2/source.go | 14 +- services/auth/source/oauth2/source_callout.go | 4 +- .../auth/source/oauth2/source_register.go | 6 +- .../auth/source/pam/assert_interface_test.go | 6 +- services/auth/source/pam/source.go | 14 +- .../auth/source/pam/source_authenticate.go | 6 +- .../auth/source/smtp/assert_interface_test.go | 12 +- services/auth/source/smtp/source.go | 14 +- .../auth/source/smtp/source_authenticate.go | 6 +- .../auth/source/sspi/assert_interface_test.go | 4 +- services/auth/source/sspi/source.go | 4 +- services/auth/sspi_windows.go | 6 +- services/auth/sync.go | 4 +- services/externalaccount/user.go | 6 +- .../accountlinks.tmpl} | 0 .../openid.tmpl} | 0 .../settings/{ => security}/security.tmpl | 8 +- .../twofa.tmpl} | 0 .../settings/{ => security}/twofa_enroll.tmpl | 0 .../{security_u2f.tmpl => security/u2f.tmpl} | 0 87 files changed, 2880 insertions(+), 2770 deletions(-) rename models/{login => auth}/main_test.go (97%) rename models/{login/oauth2_application.go => auth/oauth2.go} (90%) rename models/{login/oauth2_application_test.go => auth/oauth2_test.go} (99%) rename models/{login => auth}/session.go (99%) rename models/{login => auth}/source.go (91%) rename models/{login => auth}/source_test.go (88%) rename models/{login => auth}/twofactor.go (99%) rename models/{login => auth}/u2f.go (99%) rename models/{login => auth}/u2f_test.go (99%) delete mode 100644 models/login/oauth2.go create mode 100644 routers/web/auth/2fa.go create mode 100644 routers/web/auth/auth.go create mode 100644 routers/web/auth/linkaccount.go create mode 100644 routers/web/auth/main_test.go rename routers/web/{user => auth}/oauth.go (65%) rename routers/web/{user => auth}/oauth_test.go (90%) rename routers/web/{user/auth_openid.go => auth/openid.go} (99%) create mode 100644 routers/web/auth/password.go create mode 100644 routers/web/auth/u2f.go delete mode 100644 routers/web/user/auth.go rename routers/web/user/setting/{security_twofa.go => security/2fa.go} (91%) rename routers/web/user/setting/{security_openid.go => security/openid.go} (99%) rename routers/web/user/setting/{ => security}/security.go (76%) rename routers/web/user/setting/{security_u2f.go => security/u2f.go} (89%) rename services/auth/{login_source.go => source.go} (72%) rename templates/user/settings/{security_accountlinks.tmpl => security/accountlinks.tmpl} (100%) rename templates/user/settings/{security_openid.tmpl => security/openid.tmpl} (100%) rename templates/user/settings/{ => security}/security.tmpl (56%) rename templates/user/settings/{security_twofa.tmpl => security/twofa.tmpl} (100%) rename templates/user/settings/{ => security}/twofa_enroll.tmpl (100%) rename templates/user/settings/{security_u2f.tmpl => security/u2f.tmpl} (100%) diff --git a/cmd/admin.go b/cmd/admin.go index 099803fbf52b..ed2aa5de2338 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -15,8 +15,8 @@ import ( "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" @@ -700,8 +700,8 @@ func runAddOauth(c *cli.Context) error { return err } - return login.CreateSource(&login.Source{ - Type: login.OAuth2, + return auth.CreateSource(&auth.Source{ + Type: auth.OAuth2, Name: c.String("name"), IsActive: true, Cfg: parseOAuth2Config(c), @@ -720,7 +720,7 @@ func runUpdateOauth(c *cli.Context) error { return err } - source, err := login.GetSourceByID(c.Int64("id")) + source, err := auth.GetSourceByID(c.Int64("id")) if err != nil { return err } @@ -801,7 +801,7 @@ func runUpdateOauth(c *cli.Context) error { oAuth2Config.CustomURLMapping = customURLMapping source.Cfg = oAuth2Config - return login.UpdateSource(source) + return auth.UpdateSource(source) } func runListAuth(c *cli.Context) error { @@ -812,7 +812,7 @@ func runListAuth(c *cli.Context) error { return err } - loginSources, err := login.Sources() + authSources, err := auth.Sources() if err != nil { return err @@ -831,7 +831,7 @@ func runListAuth(c *cli.Context) error { // loop through each source and print w := tabwriter.NewWriter(os.Stdout, c.Int("min-width"), c.Int("tab-width"), c.Int("padding"), padChar, flags) fmt.Fprintf(w, "ID\tName\tType\tEnabled\n") - for _, source := range loginSources { + for _, source := range authSources { fmt.Fprintf(w, "%d\t%s\t%s\t%t\n", source.ID, source.Name, source.Type.String(), source.IsActive) } w.Flush() @@ -851,10 +851,10 @@ func runDeleteAuth(c *cli.Context) error { return err } - source, err := login.GetSourceByID(c.Int64("id")) + source, err := auth.GetSourceByID(c.Int64("id")) if err != nil { return err } - return auth_service.DeleteLoginSource(source) + return auth_service.DeleteSource(source) } diff --git a/cmd/admin_auth_ldap.go b/cmd/admin_auth_ldap.go index 950d515e3916..06f9244d5009 100644 --- a/cmd/admin_auth_ldap.go +++ b/cmd/admin_auth_ldap.go @@ -9,7 +9,7 @@ import ( "fmt" "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth/source/ldap" "github.com/urfave/cli" @@ -17,10 +17,10 @@ import ( type ( authService struct { - initDB func(ctx context.Context) error - createLoginSource func(loginSource *login.Source) error - updateLoginSource func(loginSource *login.Source) error - getLoginSourceByID func(id int64) (*login.Source, error) + initDB func(ctx context.Context) error + createAuthSource func(*auth.Source) error + updateAuthSource func(*auth.Source) error + getAuthSourceByID func(id int64) (*auth.Source, error) } ) @@ -168,23 +168,23 @@ var ( // newAuthService creates a service with default functions. func newAuthService() *authService { return &authService{ - initDB: initDB, - createLoginSource: login.CreateSource, - updateLoginSource: login.UpdateSource, - getLoginSourceByID: login.GetSourceByID, + initDB: initDB, + createAuthSource: auth.CreateSource, + updateAuthSource: auth.UpdateSource, + getAuthSourceByID: auth.GetSourceByID, } } -// parseLoginSource assigns values on loginSource according to command line flags. -func parseLoginSource(c *cli.Context, loginSource *login.Source) { +// parseAuthSource assigns values on authSource according to command line flags. +func parseAuthSource(c *cli.Context, authSource *auth.Source) { if c.IsSet("name") { - loginSource.Name = c.String("name") + authSource.Name = c.String("name") } if c.IsSet("not-active") { - loginSource.IsActive = !c.Bool("not-active") + authSource.IsActive = !c.Bool("not-active") } if c.IsSet("synchronize-users") { - loginSource.IsSyncEnabled = c.Bool("synchronize-users") + authSource.IsSyncEnabled = c.Bool("synchronize-users") } } @@ -275,23 +275,23 @@ func findLdapSecurityProtocolByName(name string) (ldap.SecurityProtocol, bool) { return 0, false } -// getLoginSource gets the login source by its id defined in the command line flags. +// getAuthSource gets the login source by its id defined in the command line flags. // It returns an error if the id is not set, does not match any source or if the source is not of expected type. -func (a *authService) getLoginSource(c *cli.Context, loginType login.Type) (*login.Source, error) { +func (a *authService) getAuthSource(c *cli.Context, authType auth.Type) (*auth.Source, error) { if err := argsSet(c, "id"); err != nil { return nil, err } - loginSource, err := a.getLoginSourceByID(c.Int64("id")) + authSource, err := a.getAuthSourceByID(c.Int64("id")) if err != nil { return nil, err } - if loginSource.Type != loginType { - return nil, fmt.Errorf("Invalid authentication type. expected: %s, actual: %s", loginType.String(), loginSource.Type.String()) + if authSource.Type != authType { + return nil, fmt.Errorf("Invalid authentication type. expected: %s, actual: %s", authType.String(), authSource.Type.String()) } - return loginSource, nil + return authSource, nil } // addLdapBindDn adds a new LDAP via Bind DN authentication source. @@ -307,20 +307,20 @@ func (a *authService) addLdapBindDn(c *cli.Context) error { return err } - loginSource := &login.Source{ - Type: login.LDAP, + authSource := &auth.Source{ + Type: auth.LDAP, IsActive: true, // active by default Cfg: &ldap.Source{ Enabled: true, // always true }, } - parseLoginSource(c, loginSource) - if err := parseLdapConfig(c, loginSource.Cfg.(*ldap.Source)); err != nil { + parseAuthSource(c, authSource) + if err := parseLdapConfig(c, authSource.Cfg.(*ldap.Source)); err != nil { return err } - return a.createLoginSource(loginSource) + return a.createAuthSource(authSource) } // updateLdapBindDn updates a new LDAP via Bind DN authentication source. @@ -332,17 +332,17 @@ func (a *authService) updateLdapBindDn(c *cli.Context) error { return err } - loginSource, err := a.getLoginSource(c, login.LDAP) + authSource, err := a.getAuthSource(c, auth.LDAP) if err != nil { return err } - parseLoginSource(c, loginSource) - if err := parseLdapConfig(c, loginSource.Cfg.(*ldap.Source)); err != nil { + parseAuthSource(c, authSource) + if err := parseLdapConfig(c, authSource.Cfg.(*ldap.Source)); err != nil { return err } - return a.updateLoginSource(loginSource) + return a.updateAuthSource(authSource) } // addLdapSimpleAuth adds a new LDAP (simple auth) authentication source. @@ -358,20 +358,20 @@ func (a *authService) addLdapSimpleAuth(c *cli.Context) error { return err } - loginSource := &login.Source{ - Type: login.DLDAP, + authSource := &auth.Source{ + Type: auth.DLDAP, IsActive: true, // active by default Cfg: &ldap.Source{ Enabled: true, // always true }, } - parseLoginSource(c, loginSource) - if err := parseLdapConfig(c, loginSource.Cfg.(*ldap.Source)); err != nil { + parseAuthSource(c, authSource) + if err := parseLdapConfig(c, authSource.Cfg.(*ldap.Source)); err != nil { return err } - return a.createLoginSource(loginSource) + return a.createAuthSource(authSource) } // updateLdapBindDn updates a new LDAP (simple auth) authentication source. @@ -383,15 +383,15 @@ func (a *authService) updateLdapSimpleAuth(c *cli.Context) error { return err } - loginSource, err := a.getLoginSource(c, login.DLDAP) + authSource, err := a.getAuthSource(c, auth.DLDAP) if err != nil { return err } - parseLoginSource(c, loginSource) - if err := parseLdapConfig(c, loginSource.Cfg.(*ldap.Source)); err != nil { + parseAuthSource(c, authSource) + if err := parseLdapConfig(c, authSource.Cfg.(*ldap.Source)); err != nil { return err } - return a.updateLoginSource(loginSource) + return a.updateAuthSource(authSource) } diff --git a/cmd/admin_auth_ldap_test.go b/cmd/admin_auth_ldap_test.go index 15880639d6e5..e1cd1c3244eb 100644 --- a/cmd/admin_auth_ldap_test.go +++ b/cmd/admin_auth_ldap_test.go @@ -8,7 +8,7 @@ import ( "context" "testing" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth/source/ldap" "github.com/stretchr/testify/assert" @@ -23,9 +23,9 @@ func TestAddLdapBindDn(t *testing.T) { // Test cases var cases = []struct { - args []string - loginSource *login.Source - errMsg string + args []string + source *auth.Source + errMsg string }{ // case 0 { @@ -53,8 +53,8 @@ func TestAddLdapBindDn(t *testing.T) { "--synchronize-users", "--page-size", "99", }, - loginSource: &login.Source{ - Type: login.LDAP, + source: &auth.Source{ + Type: auth.LDAP, Name: "ldap (via Bind DN) source full", IsActive: false, IsSyncEnabled: true, @@ -94,8 +94,8 @@ func TestAddLdapBindDn(t *testing.T) { "--user-filter", "(memberOf=cn=user-group,ou=example,dc=min-domain-bind,dc=org)", "--email-attribute", "mail-bind min", }, - loginSource: &login.Source{ - Type: login.LDAP, + source: &auth.Source{ + Type: auth.LDAP, Name: "ldap (via Bind DN) source min", IsActive: true, Cfg: &ldap.Source{ @@ -206,21 +206,21 @@ func TestAddLdapBindDn(t *testing.T) { for n, c := range cases { // Mock functions. - var createdLoginSource *login.Source + var createdAuthSource *auth.Source service := &authService{ initDB: func(context.Context) error { return nil }, - createLoginSource: func(loginSource *login.Source) error { - createdLoginSource = loginSource + createAuthSource: func(authSource *auth.Source) error { + createdAuthSource = authSource return nil }, - updateLoginSource: func(loginSource *login.Source) error { - assert.FailNow(t, "case %d: should not call updateLoginSource", n) + updateAuthSource: func(authSource *auth.Source) error { + assert.FailNow(t, "case %d: should not call updateAuthSource", n) return nil }, - getLoginSourceByID: func(id int64) (*login.Source, error) { - assert.FailNow(t, "case %d: should not call getLoginSourceByID", n) + getAuthSourceByID: func(id int64) (*auth.Source, error) { + assert.FailNow(t, "case %d: should not call getAuthSourceByID", n) return nil, nil }, } @@ -236,7 +236,7 @@ func TestAddLdapBindDn(t *testing.T) { assert.EqualError(t, err, c.errMsg, "case %d: error should match", n) } else { assert.NoError(t, err, "case %d: should have no errors", n) - assert.Equal(t, c.loginSource, createdLoginSource, "case %d: wrong loginSource", n) + assert.Equal(t, c.source, createdAuthSource, "case %d: wrong authSource", n) } } } @@ -249,9 +249,9 @@ func TestAddLdapSimpleAuth(t *testing.T) { // Test cases var cases = []struct { - args []string - loginSource *login.Source - errMsg string + args []string + authSource *auth.Source + errMsg string }{ // case 0 { @@ -275,8 +275,8 @@ func TestAddLdapSimpleAuth(t *testing.T) { "--avatar-attribute", "avatar-simple full", "--user-dn", "cn=%s,ou=Users,dc=full-domain-simple,dc=org", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Name: "ldap (simple auth) source full", IsActive: false, Cfg: &ldap.Source{ @@ -312,8 +312,8 @@ func TestAddLdapSimpleAuth(t *testing.T) { "--email-attribute", "mail-simple min", "--user-dn", "cn=%s,ou=Users,dc=min-domain-simple,dc=org", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Name: "ldap (simple auth) source min", IsActive: true, Cfg: &ldap.Source{ @@ -437,21 +437,21 @@ func TestAddLdapSimpleAuth(t *testing.T) { for n, c := range cases { // Mock functions. - var createdLoginSource *login.Source + var createdAuthSource *auth.Source service := &authService{ initDB: func(context.Context) error { return nil }, - createLoginSource: func(loginSource *login.Source) error { - createdLoginSource = loginSource + createAuthSource: func(authSource *auth.Source) error { + createdAuthSource = authSource return nil }, - updateLoginSource: func(loginSource *login.Source) error { - assert.FailNow(t, "case %d: should not call updateLoginSource", n) + updateAuthSource: func(authSource *auth.Source) error { + assert.FailNow(t, "case %d: should not call updateAuthSource", n) return nil }, - getLoginSourceByID: func(id int64) (*login.Source, error) { - assert.FailNow(t, "case %d: should not call getLoginSourceByID", n) + getAuthSourceByID: func(id int64) (*auth.Source, error) { + assert.FailNow(t, "case %d: should not call getAuthSourceByID", n) return nil, nil }, } @@ -467,7 +467,7 @@ func TestAddLdapSimpleAuth(t *testing.T) { assert.EqualError(t, err, c.errMsg, "case %d: error should match", n) } else { assert.NoError(t, err, "case %d: should have no errors", n) - assert.Equal(t, c.loginSource, createdLoginSource, "case %d: wrong loginSource", n) + assert.Equal(t, c.authSource, createdAuthSource, "case %d: wrong authSource", n) } } } @@ -480,11 +480,11 @@ func TestUpdateLdapBindDn(t *testing.T) { // Test cases var cases = []struct { - args []string - id int64 - existingLoginSource *login.Source - loginSource *login.Source - errMsg string + args []string + id int64 + existingAuthSource *auth.Source + authSource *auth.Source + errMsg string }{ // case 0 { @@ -513,15 +513,15 @@ func TestUpdateLdapBindDn(t *testing.T) { "--page-size", "99", }, id: 23, - existingLoginSource: &login.Source{ - Type: login.LDAP, + existingAuthSource: &auth.Source{ + Type: auth.LDAP, IsActive: true, Cfg: &ldap.Source{ Enabled: true, }, }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Name: "ldap (via Bind DN) source full", IsActive: false, IsSyncEnabled: true, @@ -555,8 +555,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "ldap-test", "--id", "1", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{}, }, }, @@ -567,8 +567,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--name", "ldap (via Bind DN) source", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Name: "ldap (via Bind DN) source", Cfg: &ldap.Source{ Name: "ldap (via Bind DN) source", @@ -582,13 +582,13 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--not-active", }, - existingLoginSource: &login.Source{ - Type: login.LDAP, + existingAuthSource: &auth.Source{ + Type: auth.LDAP, IsActive: true, Cfg: &ldap.Source{}, }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, IsActive: false, Cfg: &ldap.Source{}, }, @@ -600,8 +600,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--security-protocol", "LDAPS", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ SecurityProtocol: ldap.SecurityProtocol(1), }, @@ -614,8 +614,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--skip-tls-verify", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ SkipVerify: true, }, @@ -628,8 +628,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--host", "ldap-server", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ Host: "ldap-server", }, @@ -642,8 +642,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--port", "389", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ Port: 389, }, @@ -656,8 +656,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--user-search-base", "ou=Users,dc=domain,dc=org", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ UserBase: "ou=Users,dc=domain,dc=org", }, @@ -670,8 +670,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--user-filter", "(memberOf=cn=user-group,ou=example,dc=domain,dc=org)", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ Filter: "(memberOf=cn=user-group,ou=example,dc=domain,dc=org)", }, @@ -684,8 +684,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--admin-filter", "(memberOf=cn=admin-group,ou=example,dc=domain,dc=org)", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AdminFilter: "(memberOf=cn=admin-group,ou=example,dc=domain,dc=org)", }, @@ -698,8 +698,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--username-attribute", "uid", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributeUsername: "uid", }, @@ -712,8 +712,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--firstname-attribute", "givenName", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributeName: "givenName", }, @@ -726,8 +726,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--surname-attribute", "sn", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributeSurname: "sn", }, @@ -740,8 +740,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--email-attribute", "mail", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributeMail: "mail", }, @@ -754,8 +754,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--attributes-in-bind", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributesInBind: true, }, @@ -768,8 +768,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--public-ssh-key-attribute", "publickey", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ AttributeSSHPublicKey: "publickey", }, @@ -782,8 +782,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--bind-dn", "cn=readonly,dc=domain,dc=org", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ BindDN: "cn=readonly,dc=domain,dc=org", }, @@ -796,8 +796,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--bind-password", "secret", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ BindPassword: "secret", }, @@ -810,8 +810,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--synchronize-users", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, IsSyncEnabled: true, Cfg: &ldap.Source{}, }, @@ -823,8 +823,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "--id", "1", "--page-size", "12", }, - loginSource: &login.Source{ - Type: login.LDAP, + authSource: &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{ SearchPageSize: 12, }, @@ -852,8 +852,8 @@ func TestUpdateLdapBindDn(t *testing.T) { "ldap-test", "--id", "1", }, - existingLoginSource: &login.Source{ - Type: login.OAuth2, + existingAuthSource: &auth.Source{ + Type: auth.OAuth2, Cfg: &ldap.Source{}, }, errMsg: "Invalid authentication type. expected: LDAP (via BindDN), actual: OAuth2", @@ -862,28 +862,28 @@ func TestUpdateLdapBindDn(t *testing.T) { for n, c := range cases { // Mock functions. - var updatedLoginSource *login.Source + var updatedAuthSource *auth.Source service := &authService{ initDB: func(context.Context) error { return nil }, - createLoginSource: func(loginSource *login.Source) error { - assert.FailNow(t, "case %d: should not call createLoginSource", n) + createAuthSource: func(authSource *auth.Source) error { + assert.FailNow(t, "case %d: should not call createAuthSource", n) return nil }, - updateLoginSource: func(loginSource *login.Source) error { - updatedLoginSource = loginSource + updateAuthSource: func(authSource *auth.Source) error { + updatedAuthSource = authSource return nil }, - getLoginSourceByID: func(id int64) (*login.Source, error) { + getAuthSourceByID: func(id int64) (*auth.Source, error) { if c.id != 0 { assert.Equal(t, c.id, id, "case %d: wrong id", n) } - if c.existingLoginSource != nil { - return c.existingLoginSource, nil + if c.existingAuthSource != nil { + return c.existingAuthSource, nil } - return &login.Source{ - Type: login.LDAP, + return &auth.Source{ + Type: auth.LDAP, Cfg: &ldap.Source{}, }, nil }, @@ -900,7 +900,7 @@ func TestUpdateLdapBindDn(t *testing.T) { assert.EqualError(t, err, c.errMsg, "case %d: error should match", n) } else { assert.NoError(t, err, "case %d: should have no errors", n) - assert.Equal(t, c.loginSource, updatedLoginSource, "case %d: wrong loginSource", n) + assert.Equal(t, c.authSource, updatedAuthSource, "case %d: wrong authSource", n) } } } @@ -913,11 +913,11 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { // Test cases var cases = []struct { - args []string - id int64 - existingLoginSource *login.Source - loginSource *login.Source - errMsg string + args []string + id int64 + existingAuthSource *auth.Source + authSource *auth.Source + errMsg string }{ // case 0 { @@ -943,8 +943,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--user-dn", "cn=%s,ou=Users,dc=full-domain-simple,dc=org", }, id: 7, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Name: "ldap (simple auth) source full", IsActive: false, Cfg: &ldap.Source{ @@ -973,8 +973,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "ldap-test", "--id", "1", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{}, }, }, @@ -985,8 +985,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--name", "ldap (simple auth) source", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Name: "ldap (simple auth) source", Cfg: &ldap.Source{ Name: "ldap (simple auth) source", @@ -1000,13 +1000,13 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--not-active", }, - existingLoginSource: &login.Source{ - Type: login.DLDAP, + existingAuthSource: &auth.Source{ + Type: auth.DLDAP, IsActive: true, Cfg: &ldap.Source{}, }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, IsActive: false, Cfg: &ldap.Source{}, }, @@ -1018,8 +1018,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--security-protocol", "starttls", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ SecurityProtocol: ldap.SecurityProtocol(2), }, @@ -1032,8 +1032,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--skip-tls-verify", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ SkipVerify: true, }, @@ -1046,8 +1046,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--host", "ldap-server", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ Host: "ldap-server", }, @@ -1060,8 +1060,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--port", "987", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ Port: 987, }, @@ -1074,8 +1074,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--user-search-base", "ou=Users,dc=domain,dc=org", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ UserBase: "ou=Users,dc=domain,dc=org", }, @@ -1088,8 +1088,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--user-filter", "(&(objectClass=posixAccount)(cn=%s))", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ Filter: "(&(objectClass=posixAccount)(cn=%s))", }, @@ -1102,8 +1102,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--admin-filter", "(memberOf=cn=admin-group,ou=example,dc=domain,dc=org)", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AdminFilter: "(memberOf=cn=admin-group,ou=example,dc=domain,dc=org)", }, @@ -1116,8 +1116,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--username-attribute", "uid", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AttributeUsername: "uid", }, @@ -1130,8 +1130,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--firstname-attribute", "givenName", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AttributeName: "givenName", }, @@ -1144,8 +1144,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--surname-attribute", "sn", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AttributeSurname: "sn", }, @@ -1158,8 +1158,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--email-attribute", "mail", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AttributeMail: "mail", @@ -1173,8 +1173,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--public-ssh-key-attribute", "publickey", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ AttributeSSHPublicKey: "publickey", }, @@ -1187,8 +1187,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "--id", "1", "--user-dn", "cn=%s,ou=Users,dc=domain,dc=org", }, - loginSource: &login.Source{ - Type: login.DLDAP, + authSource: &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{ UserDN: "cn=%s,ou=Users,dc=domain,dc=org", }, @@ -1216,8 +1216,8 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { "ldap-test", "--id", "1", }, - existingLoginSource: &login.Source{ - Type: login.PAM, + existingAuthSource: &auth.Source{ + Type: auth.PAM, Cfg: &ldap.Source{}, }, errMsg: "Invalid authentication type. expected: LDAP (simple auth), actual: PAM", @@ -1226,28 +1226,28 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { for n, c := range cases { // Mock functions. - var updatedLoginSource *login.Source + var updatedAuthSource *auth.Source service := &authService{ initDB: func(context.Context) error { return nil }, - createLoginSource: func(loginSource *login.Source) error { - assert.FailNow(t, "case %d: should not call createLoginSource", n) + createAuthSource: func(authSource *auth.Source) error { + assert.FailNow(t, "case %d: should not call createAuthSource", n) return nil }, - updateLoginSource: func(loginSource *login.Source) error { - updatedLoginSource = loginSource + updateAuthSource: func(authSource *auth.Source) error { + updatedAuthSource = authSource return nil }, - getLoginSourceByID: func(id int64) (*login.Source, error) { + getAuthSourceByID: func(id int64) (*auth.Source, error) { if c.id != 0 { assert.Equal(t, c.id, id, "case %d: wrong id", n) } - if c.existingLoginSource != nil { - return c.existingLoginSource, nil + if c.existingAuthSource != nil { + return c.existingAuthSource, nil } - return &login.Source{ - Type: login.DLDAP, + return &auth.Source{ + Type: auth.DLDAP, Cfg: &ldap.Source{}, }, nil }, @@ -1264,7 +1264,7 @@ func TestUpdateLdapSimpleAuth(t *testing.T) { assert.EqualError(t, err, c.errMsg, "case %d: error should match", n) } else { assert.NoError(t, err, "case %d: should have no errors", n) - assert.Equal(t, c.loginSource, updatedLoginSource, "case %d: wrong loginSource", n) + assert.Equal(t, c.authSource, updatedAuthSource, "case %d: wrong authSource", n) } } } diff --git a/integrations/api_oauth2_apps_test.go b/integrations/api_oauth2_apps_test.go index 52922b98485c..e51549568a6a 100644 --- a/integrations/api_oauth2_apps_test.go +++ b/integrations/api_oauth2_apps_test.go @@ -9,7 +9,7 @@ import ( "net/http" "testing" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" api "code.gitea.io/gitea/modules/structs" @@ -47,7 +47,7 @@ func testAPICreateOAuth2Application(t *testing.T) { assert.Len(t, createdApp.ClientID, 36) assert.NotEmpty(t, createdApp.Created) assert.EqualValues(t, appBody.RedirectURIs[0], createdApp.RedirectURIs[0]) - unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{UID: user.ID, Name: createdApp.Name}) + unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{UID: user.ID, Name: createdApp.Name}) } func testAPIListOAuth2Applications(t *testing.T) { @@ -55,13 +55,13 @@ func testAPIListOAuth2Applications(t *testing.T) { session := loginUser(t, user.Name) token := getTokenForLoggedInUser(t, session) - existApp := unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ + existApp := unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ UID: user.ID, Name: "test-app-1", RedirectURIs: []string{ "http://www.google.com", }, - }).(*login.OAuth2Application) + }).(*auth.OAuth2Application) urlStr := fmt.Sprintf("/api/v1/user/applications/oauth2?token=%s", token) req := NewRequest(t, "GET", urlStr) @@ -76,7 +76,7 @@ func testAPIListOAuth2Applications(t *testing.T) { assert.Len(t, expectedApp.ClientID, 36) assert.Empty(t, expectedApp.ClientSecret) assert.EqualValues(t, existApp.RedirectURIs[0], expectedApp.RedirectURIs[0]) - unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) + unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) } func testAPIDeleteOAuth2Application(t *testing.T) { @@ -84,16 +84,16 @@ func testAPIDeleteOAuth2Application(t *testing.T) { session := loginUser(t, user.Name) token := getTokenForLoggedInUser(t, session) - oldApp := unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ + oldApp := unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ UID: user.ID, Name: "test-app-1", - }).(*login.OAuth2Application) + }).(*auth.OAuth2Application) urlStr := fmt.Sprintf("/api/v1/user/applications/oauth2/%d?token=%s", oldApp.ID, token) req := NewRequest(t, "DELETE", urlStr) session.MakeRequest(t, req, http.StatusNoContent) - unittest.AssertNotExistsBean(t, &login.OAuth2Application{UID: oldApp.UID, Name: oldApp.Name}) + unittest.AssertNotExistsBean(t, &auth.OAuth2Application{UID: oldApp.UID, Name: oldApp.Name}) // Delete again will return not found req = NewRequest(t, "DELETE", urlStr) @@ -105,13 +105,13 @@ func testAPIGetOAuth2Application(t *testing.T) { session := loginUser(t, user.Name) token := getTokenForLoggedInUser(t, session) - existApp := unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ + existApp := unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ UID: user.ID, Name: "test-app-1", RedirectURIs: []string{ "http://www.google.com", }, - }).(*login.OAuth2Application) + }).(*auth.OAuth2Application) urlStr := fmt.Sprintf("/api/v1/user/applications/oauth2/%d?token=%s", existApp.ID, token) req := NewRequest(t, "GET", urlStr) @@ -127,19 +127,19 @@ func testAPIGetOAuth2Application(t *testing.T) { assert.Empty(t, expectedApp.ClientSecret) assert.Len(t, expectedApp.RedirectURIs, 1) assert.EqualValues(t, existApp.RedirectURIs[0], expectedApp.RedirectURIs[0]) - unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) + unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) } func testAPIUpdateOAuth2Application(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) - existApp := unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ + existApp := unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ UID: user.ID, Name: "test-app-1", RedirectURIs: []string{ "http://www.google.com", }, - }).(*login.OAuth2Application) + }).(*auth.OAuth2Application) appBody := api.CreateOAuth2ApplicationOptions{ Name: "test-app-1", @@ -161,5 +161,5 @@ func testAPIUpdateOAuth2Application(t *testing.T) { assert.Len(t, expectedApp.RedirectURIs, 2) assert.EqualValues(t, expectedApp.RedirectURIs[0], appBody.RedirectURIs[0]) assert.EqualValues(t, expectedApp.RedirectURIs[1], appBody.RedirectURIs[1]) - unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) + unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ID: expectedApp.ID, Name: expectedApp.Name}) } diff --git a/models/asymkey/ssh_key.go b/models/asymkey/ssh_key.go index 12ed611bc092..74d2411dd9ed 100644 --- a/models/asymkey/ssh_key.go +++ b/models/asymkey/ssh_key.go @@ -11,8 +11,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/models/perm" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" @@ -92,7 +92,7 @@ func addKey(e db.Engine, key *PublicKey) (err error) { } // AddPublicKey adds new public key to database and authorized_keys file. -func AddPublicKey(ownerID int64, name, content string, loginSourceID int64) (*PublicKey, error) { +func AddPublicKey(ownerID int64, name, content string, authSourceID int64) (*PublicKey, error) { log.Trace(content) fingerprint, err := calcFingerprint(content) @@ -128,7 +128,7 @@ func AddPublicKey(ownerID int64, name, content string, loginSourceID int64) (*Pu Content: content, Mode: perm.AccessModeWrite, Type: KeyTypeUser, - LoginSourceID: loginSourceID, + LoginSourceID: authSourceID, } if err = addKey(sess, key); err != nil { return nil, fmt.Errorf("addKey: %v", err) @@ -223,10 +223,10 @@ func CountPublicKeys(userID int64) (int64, error) { } // ListPublicKeysBySource returns a list of synchronized public keys for a given user and login source. -func ListPublicKeysBySource(uid, loginSourceID int64) ([]*PublicKey, error) { +func ListPublicKeysBySource(uid, authSourceID int64) ([]*PublicKey, error) { keys := make([]*PublicKey, 0, 5) return keys, db.GetEngine(db.DefaultContext). - Where("owner_id = ? AND login_source_id = ?", uid, loginSourceID). + Where("owner_id = ? AND login_source_id = ?", uid, authSourceID). Find(&keys) } @@ -261,7 +261,7 @@ func DeletePublicKeys(ctx context.Context, keyIDs ...int64) error { // PublicKeysAreExternallyManaged returns whether the provided KeyID represents an externally managed Key func PublicKeysAreExternallyManaged(keys []*PublicKey) ([]bool, error) { - sources := make([]*login.Source, 0, 5) + sources := make([]*auth.Source, 0, 5) externals := make([]bool, len(keys)) keyloop: for i, key := range keys { @@ -270,7 +270,7 @@ keyloop: continue keyloop } - var source *login.Source + var source *auth.Source sourceloop: for _, s := range sources { @@ -282,11 +282,11 @@ keyloop: if source == nil { var err error - source, err = login.GetSourceByID(key.LoginSourceID) + source, err = auth.GetSourceByID(key.LoginSourceID) if err != nil { - if login.IsErrSourceNotExist(err) { + if auth.IsErrSourceNotExist(err) { externals[i] = false - sources[i] = &login.Source{ + sources[i] = &auth.Source{ ID: key.LoginSourceID, } continue keyloop @@ -295,7 +295,7 @@ keyloop: } } - if sshKeyProvider, ok := source.Cfg.(login.SSHKeyProvider); ok && sshKeyProvider.ProvidesSSHKeys() { + if sshKeyProvider, ok := source.Cfg.(auth.SSHKeyProvider); ok && sshKeyProvider.ProvidesSSHKeys() { // Disable setting SSH keys for this user externals[i] = true } @@ -313,14 +313,14 @@ func PublicKeyIsExternallyManaged(id int64) (bool, error) { if key.LoginSourceID == 0 { return false, nil } - source, err := login.GetSourceByID(key.LoginSourceID) + source, err := auth.GetSourceByID(key.LoginSourceID) if err != nil { - if login.IsErrSourceNotExist(err) { + if auth.IsErrSourceNotExist(err) { return false, nil } return false, err } - if sshKeyProvider, ok := source.Cfg.(login.SSHKeyProvider); ok && sshKeyProvider.ProvidesSSHKeys() { + if sshKeyProvider, ok := source.Cfg.(auth.SSHKeyProvider); ok && sshKeyProvider.ProvidesSSHKeys() { // Disable setting SSH keys for this user return true, nil } @@ -360,7 +360,7 @@ func deleteKeysMarkedForDeletion(keys []string) (bool, error) { } // AddPublicKeysBySource add a users public keys. Returns true if there are changes. -func AddPublicKeysBySource(usr *user_model.User, s *login.Source, sshPublicKeys []string) bool { +func AddPublicKeysBySource(usr *user_model.User, s *auth.Source, sshPublicKeys []string) bool { var sshKeysNeedUpdate bool for _, sshKey := range sshPublicKeys { var err error @@ -398,7 +398,7 @@ func AddPublicKeysBySource(usr *user_model.User, s *login.Source, sshPublicKeys } // SynchronizePublicKeys updates a users public keys. Returns true if there are changes. -func SynchronizePublicKeys(usr *user_model.User, s *login.Source, sshPublicKeys []string) bool { +func SynchronizePublicKeys(usr *user_model.User, s *auth.Source, sshPublicKeys []string) bool { var sshKeysNeedUpdate bool log.Trace("synchronizePublicKeys[%s]: Handling Public SSH Key synchronization for user %s", s.Name, usr.Name) diff --git a/models/asymkey/ssh_key_principals.go b/models/asymkey/ssh_key_principals.go index 19fc6644cba7..5f18fd04d14d 100644 --- a/models/asymkey/ssh_key_principals.go +++ b/models/asymkey/ssh_key_principals.go @@ -25,7 +25,7 @@ import ( // This file contains functions related to principals // AddPrincipalKey adds new principal to database and authorized_principals file. -func AddPrincipalKey(ownerID int64, content string, loginSourceID int64) (*PublicKey, error) { +func AddPrincipalKey(ownerID int64, content string, authSourceID int64) (*PublicKey, error) { ctx, committer, err := db.TxContext() if err != nil { return nil, err @@ -49,7 +49,7 @@ func AddPrincipalKey(ownerID int64, content string, loginSourceID int64) (*Publi Content: content, Mode: perm.AccessModeWrite, Type: KeyTypePrincipal, - LoginSourceID: loginSourceID, + LoginSourceID: authSourceID, } if err = addPrincipalKey(sess, key); err != nil { return nil, fmt.Errorf("addKey: %v", err) diff --git a/models/login/main_test.go b/models/auth/main_test.go similarity index 97% rename from models/login/main_test.go rename to models/auth/main_test.go index 0666eeaad018..94a1f405d93c 100644 --- a/models/login/main_test.go +++ b/models/auth/main_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "path/filepath" diff --git a/models/login/oauth2_application.go b/models/auth/oauth2.go similarity index 90% rename from models/login/oauth2_application.go rename to models/auth/oauth2.go index 4cd73cf3a2c7..e7030fce2898 100644 --- a/models/login/oauth2_application.go +++ b/models/auth/oauth2.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "crypto/sha256" @@ -510,3 +510,55 @@ func revokeOAuth2Grant(e db.Engine, grantID, userID int64) error { _, err := e.Delete(&OAuth2Grant{ID: grantID, UserID: userID}) return err } + +// ErrOAuthClientIDInvalid will be thrown if client id cannot be found +type ErrOAuthClientIDInvalid struct { + ClientID string +} + +// IsErrOauthClientIDInvalid checks if an error is a ErrReviewNotExist. +func IsErrOauthClientIDInvalid(err error) bool { + _, ok := err.(ErrOAuthClientIDInvalid) + return ok +} + +// Error returns the error message +func (err ErrOAuthClientIDInvalid) Error() string { + return fmt.Sprintf("Client ID invalid [Client ID: %s]", err.ClientID) +} + +// ErrOAuthApplicationNotFound will be thrown if id cannot be found +type ErrOAuthApplicationNotFound struct { + ID int64 +} + +// IsErrOAuthApplicationNotFound checks if an error is a ErrReviewNotExist. +func IsErrOAuthApplicationNotFound(err error) bool { + _, ok := err.(ErrOAuthApplicationNotFound) + return ok +} + +// Error returns the error message +func (err ErrOAuthApplicationNotFound) Error() string { + return fmt.Sprintf("OAuth application not found [ID: %d]", err.ID) +} + +// GetActiveOAuth2ProviderSources returns all actived LoginOAuth2 sources +func GetActiveOAuth2ProviderSources() ([]*Source, error) { + sources := make([]*Source, 0, 1) + if err := db.GetEngine(db.DefaultContext).Where("is_active = ? and type = ?", true, OAuth2).Find(&sources); err != nil { + return nil, err + } + return sources, nil +} + +// GetActiveOAuth2SourceByName returns a OAuth2 AuthSource based on the given name +func GetActiveOAuth2SourceByName(name string) (*Source, error) { + authSource := new(Source) + has, err := db.GetEngine(db.DefaultContext).Where("name = ? and type = ? and is_active = ?", name, OAuth2, true).Get(authSource) + if !has || err != nil { + return nil, err + } + + return authSource, nil +} diff --git a/models/login/oauth2_application_test.go b/models/auth/oauth2_test.go similarity index 99% rename from models/login/oauth2_application_test.go rename to models/auth/oauth2_test.go index 47e6a27ce968..b712fc285f36 100644 --- a/models/login/oauth2_application_test.go +++ b/models/auth/oauth2_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "testing" diff --git a/models/login/session.go b/models/auth/session.go similarity index 99% rename from models/login/session.go rename to models/auth/session.go index 805606600ad5..5b130c64b680 100644 --- a/models/login/session.go +++ b/models/auth/session.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "fmt" diff --git a/models/login/source.go b/models/auth/source.go similarity index 91% rename from models/login/source.go rename to models/auth/source.go index 1001d49b51b9..6f4f5addcb9c 100644 --- a/models/login/source.go +++ b/models/auth/source.go @@ -3,12 +3,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "fmt" "reflect" - "strconv" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/modules/log" @@ -84,10 +83,7 @@ type RegisterableSource interface { UnregisterSource() error } -// SourceSettable configurations can have their loginSource set on them -type SourceSettable interface { - SetLoginSource(*Source) -} +var registeredConfigs = map[Type]func() Config{} // RegisterTypeConfig register a config for a provided type func RegisterTypeConfig(typ Type, exemplar Config) { @@ -105,7 +101,10 @@ func RegisterTypeConfig(typ Type, exemplar Config) { } } -var registeredConfigs = map[Type]func() Config{} +// SourceSettable configurations can have their authSource set on them +type SourceSettable interface { + SetAuthSource(*Source) +} // Source represents an external way for authorizing users. type Source struct { @@ -129,30 +128,17 @@ func init() { db.RegisterModel(new(Source)) } -// Cell2Int64 converts a xorm.Cell type to int64, -// and handles possible irregular cases. -func Cell2Int64(val xorm.Cell) int64 { - switch (*val).(type) { - case []uint8: - log.Trace("Cell2Int64 ([]uint8): %v", *val) - - v, _ := strconv.ParseInt(string((*val).([]uint8)), 10, 64) - return v - } - return (*val).(int64) -} - // BeforeSet is invoked from XORM before setting the value of a field of this object. func (source *Source) BeforeSet(colName string, val xorm.Cell) { if colName == "type" { - typ := Type(Cell2Int64(val)) + typ := Type(db.Cell2Int64(val)) constructor, ok := registeredConfigs[typ] if !ok { return } source.Cfg = constructor() if settable, ok := source.Cfg.(SourceSettable); ok { - settable.SetLoginSource(source) + settable.SetAuthSource(source) } } } @@ -211,7 +197,7 @@ func (source *Source) SkipVerify() bool { return ok && skipVerifiable.IsSkipVerify() } -// CreateSource inserts a LoginSource in the DB if not already +// CreateSource inserts a AuthSource in the DB if not already // existing with the given name. func CreateSource(source *Source) error { has, err := db.GetEngine(db.DefaultContext).Where("name=?", source.Name).Exist(new(Source)) @@ -235,7 +221,7 @@ func CreateSource(source *Source) error { } if settable, ok := source.Cfg.(SourceSettable); ok { - settable.SetLoginSource(source) + settable.SetAuthSource(source) } registerableSource, ok := source.Cfg.(RegisterableSource) @@ -245,7 +231,7 @@ func CreateSource(source *Source) error { err = registerableSource.RegisterSource() if err != nil { - // remove the LoginSource in case of errors while registering configuration + // remove the AuthSource in case of errors while registering configuration if _, err := db.GetEngine(db.DefaultContext).Delete(source); err != nil { log.Error("CreateSource: Error while wrapOpenIDConnectInitializeError: %v", err) } @@ -322,11 +308,11 @@ func GetSourceByID(id int64) (*Source, error) { // UpdateSource updates a Source record in DB. func UpdateSource(source *Source) error { - var originalLoginSource *Source + var originalSource *Source if source.IsOAuth2() { // keep track of the original values so we can restore in case of errors while registering OAuth2 providers var err error - if originalLoginSource, err = GetSourceByID(source.ID); err != nil { + if originalSource, err = GetSourceByID(source.ID); err != nil { return err } } @@ -341,7 +327,7 @@ func UpdateSource(source *Source) error { } if settable, ok := source.Cfg.(SourceSettable); ok { - settable.SetLoginSource(source) + settable.SetAuthSource(source) } registerableSource, ok := source.Cfg.(RegisterableSource) @@ -352,7 +338,7 @@ func UpdateSource(source *Source) error { err = registerableSource.RegisterSource() if err != nil { // restore original values since we cannot update the provider it self - if _, err := db.GetEngine(db.DefaultContext).ID(source.ID).AllCols().Update(originalLoginSource); err != nil { + if _, err := db.GetEngine(db.DefaultContext).ID(source.ID).AllCols().Update(originalSource); err != nil { log.Error("UpdateSource: Error while wrapOpenIDConnectInitializeError: %v", err) } } diff --git a/models/login/source_test.go b/models/auth/source_test.go similarity index 88% rename from models/login/source_test.go rename to models/auth/source_test.go index e7ef7c70488c..6a8e286910a2 100644 --- a/models/login/source_test.go +++ b/models/auth/source_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "strings" @@ -34,10 +34,10 @@ func (source *TestSource) ToDB() ([]byte, error) { return json.Marshal(source) } -func TestDumpLoginSource(t *testing.T) { +func TestDumpAuthSource(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - loginSourceSchema, err := db.TableInfo(new(Source)) + authSourceSchema, err := db.TableInfo(new(Source)) assert.NoError(t, err) RegisterTypeConfig(OAuth2, new(TestSource)) @@ -54,7 +54,7 @@ func TestDumpLoginSource(t *testing.T) { sb := new(strings.Builder) - db.DumpTables([]*schemas.Table{loginSourceSchema}, sb) + db.DumpTables([]*schemas.Table{authSourceSchema}, sb) assert.Contains(t, sb.String(), `"Provider":"ConvertibleSourceName"`) } diff --git a/models/login/twofactor.go b/models/auth/twofactor.go similarity index 99% rename from models/login/twofactor.go rename to models/auth/twofactor.go index acb5e1b2d50e..883e6ce01c09 100644 --- a/models/login/twofactor.go +++ b/models/auth/twofactor.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "crypto/md5" diff --git a/models/login/u2f.go b/models/auth/u2f.go similarity index 99% rename from models/login/u2f.go rename to models/auth/u2f.go index 8cea98463f11..71943b237c88 100644 --- a/models/login/u2f.go +++ b/models/auth/u2f.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "fmt" diff --git a/models/login/u2f_test.go b/models/auth/u2f_test.go similarity index 99% rename from models/login/u2f_test.go rename to models/auth/u2f_test.go index 06a37f8132e9..32ad17839c92 100644 --- a/models/login/u2f_test.go +++ b/models/auth/u2f_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package login +package auth import ( "encoding/hex" diff --git a/models/db/convert.go b/models/db/convert.go index bf9a74a9a4ba..039a6810408c 100644 --- a/models/db/convert.go +++ b/models/db/convert.go @@ -6,9 +6,12 @@ package db import ( "fmt" + "strconv" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "xorm.io/xorm" "xorm.io/xorm/schemas" ) @@ -39,3 +42,16 @@ func ConvertUtf8ToUtf8mb4() error { return nil } + +// Cell2Int64 converts a xorm.Cell type to int64, +// and handles possible irregular cases. +func Cell2Int64(val xorm.Cell) int64 { + switch (*val).(type) { + case []uint8: + log.Trace("Cell2Int64 ([]uint8): %v", *val) + + v, _ := strconv.ParseInt(string((*val).([]uint8)), 10, 64) + return v + } + return (*val).(int64) +} diff --git a/models/login/oauth2.go b/models/login/oauth2.go deleted file mode 100644 index 45ab59dd78ca..000000000000 --- a/models/login/oauth2.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package login - -import ( - "fmt" - - "code.gitea.io/gitea/models/db" -) - -// ________ _____ __ .__ -// \_____ \ / _ \ __ ___/ |_| |__ -// / | \ / /_\ \| | \ __\ | \ -// / | \/ | \ | /| | | Y \ -// \_______ /\____|__ /____/ |__| |___| / -// \/ \/ \/ - -// ErrOAuthClientIDInvalid will be thrown if client id cannot be found -type ErrOAuthClientIDInvalid struct { - ClientID string -} - -// IsErrOauthClientIDInvalid checks if an error is a ErrReviewNotExist. -func IsErrOauthClientIDInvalid(err error) bool { - _, ok := err.(ErrOAuthClientIDInvalid) - return ok -} - -// Error returns the error message -func (err ErrOAuthClientIDInvalid) Error() string { - return fmt.Sprintf("Client ID invalid [Client ID: %s]", err.ClientID) -} - -// ErrOAuthApplicationNotFound will be thrown if id cannot be found -type ErrOAuthApplicationNotFound struct { - ID int64 -} - -// IsErrOAuthApplicationNotFound checks if an error is a ErrReviewNotExist. -func IsErrOAuthApplicationNotFound(err error) bool { - _, ok := err.(ErrOAuthApplicationNotFound) - return ok -} - -// Error returns the error message -func (err ErrOAuthApplicationNotFound) Error() string { - return fmt.Sprintf("OAuth application not found [ID: %d]", err.ID) -} - -// GetActiveOAuth2ProviderLoginSources returns all actived LoginOAuth2 sources -func GetActiveOAuth2ProviderLoginSources() ([]*Source, error) { - sources := make([]*Source, 0, 1) - if err := db.GetEngine(db.DefaultContext).Where("is_active = ? and type = ?", true, OAuth2).Find(&sources); err != nil { - return nil, err - } - return sources, nil -} - -// GetActiveOAuth2LoginSourceByName returns a OAuth2 LoginSource based on the given name -func GetActiveOAuth2LoginSourceByName(name string) (*Source, error) { - loginSource := new(Source) - has, err := db.GetEngine(db.DefaultContext).Where("name = ? and type = ? and is_active = ?", name, OAuth2, true).Get(loginSource) - if !has || err != nil { - return nil, err - } - - return loginSource, nil -} diff --git a/models/repo/repo_unit.go b/models/repo/repo_unit.go index 1957f88ff358..f526cbdf8b11 100644 --- a/models/repo/repo_unit.go +++ b/models/repo/repo_unit.go @@ -8,7 +8,6 @@ import ( "fmt" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/timeutil" @@ -170,7 +169,7 @@ func (cfg *PullRequestsConfig) AllowedMergeStyleCount() int { func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { switch colName { case "type": - switch unit.Type(login.Cell2Int64(val)) { + switch unit.Type(db.Cell2Int64(val)) { case unit.TypeCode, unit.TypeReleases, unit.TypeWiki, unit.TypeProjects: r.Config = new(UnitConfig) case unit.TypeExternalWiki: diff --git a/models/statistic.go b/models/statistic.go index f39cdd5eb759..dfe543d063d7 100644 --- a/models/statistic.go +++ b/models/statistic.go @@ -6,8 +6,8 @@ package models import ( asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" @@ -21,7 +21,7 @@ type Statistic struct { Repo, Watch, Star, Action, Access, Issue, IssueClosed, IssueOpen, Comment, Oauth, Follow, - Mirror, Release, LoginSource, Webhook, + Mirror, Release, AuthSource, Webhook, Milestone, Label, HookTask, Team, UpdateTask, Project, ProjectBoard, Attachment int64 @@ -98,7 +98,7 @@ func GetStatistic() (stats Statistic) { stats.Counter.Follow, _ = e.Count(new(user_model.Follow)) stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror)) stats.Counter.Release, _ = e.Count(new(Release)) - stats.Counter.LoginSource = login.CountSources() + stats.Counter.AuthSource = auth.CountSources() stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook)) stats.Counter.Milestone, _ = e.Count(new(Milestone)) stats.Counter.Label, _ = e.Count(new(Label)) diff --git a/models/token.go b/models/token.go index b3712fce5e5c..44428a080927 100644 --- a/models/token.go +++ b/models/token.go @@ -10,8 +10,8 @@ import ( "fmt" "time" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -68,7 +68,7 @@ func NewAccessToken(t *AccessToken) error { } t.TokenSalt = salt t.Token = base.EncodeSha1(gouuid.New().String()) - t.TokenHash = login.HashToken(t.Token, t.TokenSalt) + t.TokenHash = auth.HashToken(t.Token, t.TokenSalt) t.TokenLastEight = t.Token[len(t.Token)-8:] _, err = db.GetEngine(db.DefaultContext).Insert(t) return err @@ -130,7 +130,7 @@ func GetAccessTokenBySHA(token string) (*AccessToken, error) { } for _, t := range tokens { - tempHash := login.HashToken(token, t.TokenSalt) + tempHash := auth.HashToken(token, t.TokenSalt) if subtle.ConstantTimeCompare([]byte(t.TokenHash), []byte(tempHash)) == 1 { if successfulAccessTokenCache != nil { successfulAccessTokenCache.Add(token, t.ID) diff --git a/models/user/list.go b/models/user/list.go index 6ca4613aa783..13138b3e507e 100644 --- a/models/user/list.go +++ b/models/user/list.go @@ -7,8 +7,8 @@ package user import ( "fmt" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" ) // UserList is a list of user. @@ -40,13 +40,13 @@ func (users UserList) GetTwoFaStatus() map[int64]bool { return results } -func (users UserList) loadTwoFactorStatus(e db.Engine) (map[int64]*login.TwoFactor, error) { +func (users UserList) loadTwoFactorStatus(e db.Engine) (map[int64]*auth.TwoFactor, error) { if len(users) == 0 { return nil, nil } userIDs := users.GetUserIDs() - tokenMaps := make(map[int64]*login.TwoFactor, len(userIDs)) + tokenMaps := make(map[int64]*auth.TwoFactor, len(userIDs)) err := e. In("uid", userIDs). Find(&tokenMaps) diff --git a/models/user/user.go b/models/user/user.go index d56a225d5f83..06cacd85feb9 100644 --- a/models/user/user.go +++ b/models/user/user.go @@ -19,8 +19,8 @@ import ( _ "image/jpeg" // Needed for jpeg support + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/modules/auth/openid" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" @@ -89,7 +89,7 @@ type User struct { // is to change his/her password after registration. MustChangePassword bool `xorm:"NOT NULL DEFAULT false"` - LoginType login.Type + LoginType auth.Type LoginSource int64 `xorm:"NOT NULL DEFAULT 0"` LoginName string Type UserType @@ -232,12 +232,12 @@ func GetAllUsers() ([]*User, error) { // IsLocal returns true if user login type is LoginPlain. func (u *User) IsLocal() bool { - return u.LoginType <= login.Plain + return u.LoginType <= auth.Plain } // IsOAuth2 returns true if user login type is LoginOAuth2. func (u *User) IsOAuth2() bool { - return u.LoginType == login.OAuth2 + return u.LoginType == auth.OAuth2 } // MaxCreationLimit returns the number of repositories a user is allowed to create @@ -1012,7 +1012,7 @@ func GetUserIDsByNames(names []string, ignoreNonExistent bool) ([]int64, error) } // GetUsersBySource returns a list of Users for a login source -func GetUsersBySource(s *login.Source) ([]*User, error) { +func GetUsersBySource(s *auth.Source) ([]*User, error) { var users []*User err := db.GetEngine(db.DefaultContext).Where("login_type = ? AND login_source = ?", s.Type, s.ID).Find(&users) return users, err diff --git a/models/user/user_test.go b/models/user/user_test.go index f4acb923788f..70591c8c1213 100644 --- a/models/user/user_test.go +++ b/models/user/user_test.go @@ -9,8 +9,8 @@ import ( "strings" "testing" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -21,7 +21,7 @@ import ( func TestOAuth2Application_LoadUser(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - app := unittest.AssertExistsAndLoadBean(t, &login.OAuth2Application{ID: 1}).(*login.OAuth2Application) + app := unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ID: 1}).(*auth.OAuth2Application) user, err := GetUserByID(app.UID) assert.NoError(t, err) assert.NotNil(t, user) diff --git a/modules/context/api.go b/modules/context/api.go index 635a54c7efad..b079385affd1 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -13,13 +13,13 @@ import ( "net/url" "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web/middleware" - "code.gitea.io/gitea/services/auth" + auth_service "code.gitea.io/gitea/services/auth" "gitea.com/go-chi/session" ) @@ -225,9 +225,9 @@ func (ctx *APIContext) CheckForOTP() { } otpHeader := ctx.Req.Header.Get("X-Gitea-OTP") - twofa, err := login.GetTwoFactorByUID(ctx.Context.User.ID) + twofa, err := auth.GetTwoFactorByUID(ctx.Context.User.ID) if err != nil { - if login.IsErrTwoFactorNotEnrolled(err) { + if auth.IsErrTwoFactorNotEnrolled(err) { return // No 2FA enrollment for this user } ctx.Context.Error(http.StatusInternalServerError) @@ -244,8 +244,8 @@ func (ctx *APIContext) CheckForOTP() { } } -// APIAuth converts auth.Auth as a middleware -func APIAuth(authMethod auth.Method) func(*APIContext) { +// APIAuth converts auth_service.Auth as a middleware +func APIAuth(authMethod auth_service.Method) func(*APIContext) { return func(ctx *APIContext) { // Get user from session if logged in. ctx.User = authMethod.Verify(ctx.Req, ctx.Resp, ctx, ctx.Session) @@ -253,7 +253,7 @@ func APIAuth(authMethod auth.Method) func(*APIContext) { if ctx.Locale.Language() != ctx.User.Language { ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) } - ctx.IsBasicAuth = ctx.Data["AuthedMethod"].(string) == auth.BasicMethodName + ctx.IsBasicAuth = ctx.Data["AuthedMethod"].(string) == auth_service.BasicMethodName ctx.IsSigned = true ctx.Data["IsSigned"] = ctx.IsSigned ctx.Data["SignedUser"] = ctx.User diff --git a/modules/context/auth.go b/modules/context/auth.go index 7faa93d78b59..7e7e8ab9611c 100644 --- a/modules/context/auth.go +++ b/modules/context/auth.go @@ -8,7 +8,7 @@ package context import ( "net/http" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web/middleware" @@ -154,9 +154,9 @@ func ToggleAPI(options *ToggleOptions) func(ctx *APIContext) { if skip, ok := ctx.Data["SkipLocalTwoFA"]; ok && skip.(bool) { return // Skip 2FA } - twofa, err := login.GetTwoFactorByUID(ctx.User.ID) + twofa, err := auth.GetTwoFactorByUID(ctx.User.ID) if err != nil { - if login.IsErrTwoFactorNotEnrolled(err) { + if auth.IsErrTwoFactorNotEnrolled(err) { return // No 2FA enrollment for this user } ctx.InternalServerError(err) diff --git a/modules/convert/convert.go b/modules/convert/convert.go index 86e1c69d3656..f2b62a74bf7c 100644 --- a/modules/convert/convert.go +++ b/modules/convert/convert.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -344,8 +344,8 @@ func ToTopicResponse(topic *repo_model.Topic) *api.TopicResponse { } } -// ToOAuth2Application convert from login.OAuth2Application to api.OAuth2Application -func ToOAuth2Application(app *login.OAuth2Application) *api.OAuth2Application { +// ToOAuth2Application convert from auth.OAuth2Application to api.OAuth2Application +func ToOAuth2Application(app *auth.OAuth2Application) *api.OAuth2Application { return &api.OAuth2Application{ ID: app.ID, Name: app.Name, diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go index 527202e0a6d6..646311858171 100755 --- a/modules/metrics/collector.go +++ b/modules/metrics/collector.go @@ -297,7 +297,7 @@ func (c Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.LoginSources, prometheus.GaugeValue, - float64(stats.Counter.LoginSource), + float64(stats.Counter.AuthSource), ) ch <- prometheus.MustNewConstMetric( c.Milestones, diff --git a/modules/session/db.go b/modules/session/db.go index bc4aeb08cd8f..7ba27cba23d2 100644 --- a/modules/session/db.go +++ b/modules/session/db.go @@ -8,7 +8,7 @@ import ( "log" "sync" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/timeutil" "gitea.com/go-chi/session" @@ -72,7 +72,7 @@ func (s *DBStore) Release() error { return err } - return login.UpdateSession(s.sid, data) + return auth.UpdateSession(s.sid, data) } // Flush deletes all session data. @@ -98,7 +98,7 @@ func (p *DBProvider) Init(maxLifetime int64, connStr string) error { // Read returns raw session store by session ID. func (p *DBProvider) Read(sid string) (session.RawStore, error) { - s, err := login.ReadSession(sid) + s, err := auth.ReadSession(sid) if err != nil { return nil, err } @@ -118,7 +118,7 @@ func (p *DBProvider) Read(sid string) (session.RawStore, error) { // Exist returns true if session with given ID exists. func (p *DBProvider) Exist(sid string) bool { - has, err := login.ExistSession(sid) + has, err := auth.ExistSession(sid) if err != nil { panic("session/DB: error checking existence: " + err.Error()) } @@ -127,12 +127,12 @@ func (p *DBProvider) Exist(sid string) bool { // Destroy deletes a session by session ID. func (p *DBProvider) Destroy(sid string) error { - return login.DestroySession(sid) + return auth.DestroySession(sid) } // Regenerate regenerates a session store from old session ID to new one. func (p *DBProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) { - s, err := login.RegenerateSession(oldsid, sid) + s, err := auth.RegenerateSession(oldsid, sid) if err != nil { return nil, err @@ -153,7 +153,7 @@ func (p *DBProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err err // Count counts and returns number of sessions. func (p *DBProvider) Count() int { - total, err := login.CountSessions() + total, err := auth.CountSessions() if err != nil { panic("session/DB: error counting records: " + err.Error()) } @@ -162,7 +162,7 @@ func (p *DBProvider) Count() int { // GC calls GC to clean expired sessions. func (p *DBProvider) GC() { - if err := login.CleanupSessions(p.maxLifetime); err != nil { + if err := auth.CleanupSessions(p.maxLifetime); err != nil { log.Printf("session/DB: error garbage collecting: %v", err) } } diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index 5d2bbdea2f41..e95ab33d77d5 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -13,8 +13,8 @@ import ( "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" @@ -30,17 +30,17 @@ import ( user_service "code.gitea.io/gitea/services/user" ) -func parseLoginSource(ctx *context.APIContext, u *user_model.User, sourceID int64, loginName string) { +func parseAuthSource(ctx *context.APIContext, u *user_model.User, sourceID int64, loginName string) { if sourceID == 0 { return } - source, err := login.GetSourceByID(sourceID) + source, err := auth.GetSourceByID(sourceID) if err != nil { - if login.IsErrSourceNotExist(err) { + if auth.IsErrSourceNotExist(err) { ctx.Error(http.StatusUnprocessableEntity, "", err) } else { - ctx.Error(http.StatusInternalServerError, "login.GetSourceByID", err) + ctx.Error(http.StatusInternalServerError, "auth.GetSourceByID", err) } return } @@ -82,13 +82,13 @@ func CreateUser(ctx *context.APIContext) { Passwd: form.Password, MustChangePassword: true, IsActive: true, - LoginType: login.Plain, + LoginType: auth.Plain, } if form.MustChangePassword != nil { u.MustChangePassword = *form.MustChangePassword } - parseLoginSource(ctx, u, form.SourceID, form.LoginName) + parseAuthSource(ctx, u, form.SourceID, form.LoginName) if ctx.Written() { return } @@ -168,7 +168,7 @@ func EditUser(ctx *context.APIContext) { return } - parseLoginSource(ctx, u, form.SourceID, form.LoginName) + parseAuthSource(ctx, u, form.SourceID, form.LoginName) if ctx.Written() { return } diff --git a/routers/api/v1/user/app.go b/routers/api/v1/user/app.go index bf45bf4dd5a6..94cfab45bdca 100644 --- a/routers/api/v1/user/app.go +++ b/routers/api/v1/user/app.go @@ -12,7 +12,7 @@ import ( "strconv" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" api "code.gitea.io/gitea/modules/structs" @@ -213,7 +213,7 @@ func CreateOauth2Application(ctx *context.APIContext) { data := web.GetForm(ctx).(*api.CreateOAuth2ApplicationOptions) - app, err := login.CreateOAuth2Application(login.CreateOAuth2ApplicationOptions{ + app, err := auth.CreateOAuth2Application(auth.CreateOAuth2ApplicationOptions{ Name: data.Name, UserID: ctx.User.ID, RedirectURIs: data.RedirectURIs, @@ -252,7 +252,7 @@ func ListOauth2Applications(ctx *context.APIContext) { // "200": // "$ref": "#/responses/OAuth2ApplicationList" - apps, total, err := login.ListOAuth2Applications(ctx.User.ID, utils.GetListOptions(ctx)) + apps, total, err := auth.ListOAuth2Applications(ctx.User.ID, utils.GetListOptions(ctx)) if err != nil { ctx.Error(http.StatusInternalServerError, "ListOAuth2Applications", err) return @@ -288,8 +288,8 @@ func DeleteOauth2Application(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" appID := ctx.ParamsInt64(":id") - if err := login.DeleteOAuth2Application(appID, ctx.User.ID); err != nil { - if login.IsErrOAuthApplicationNotFound(err) { + if err := auth.DeleteOAuth2Application(appID, ctx.User.ID); err != nil { + if auth.IsErrOAuthApplicationNotFound(err) { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "DeleteOauth2ApplicationByID", err) @@ -320,9 +320,9 @@ func GetOauth2Application(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" appID := ctx.ParamsInt64(":id") - app, err := login.GetOAuth2ApplicationByID(appID) + app, err := auth.GetOAuth2ApplicationByID(appID) if err != nil { - if login.IsErrOauthClientIDInvalid(err) || login.IsErrOAuthApplicationNotFound(err) { + if auth.IsErrOauthClientIDInvalid(err) || auth.IsErrOAuthApplicationNotFound(err) { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "GetOauth2ApplicationByID", err) @@ -363,14 +363,14 @@ func UpdateOauth2Application(ctx *context.APIContext) { data := web.GetForm(ctx).(*api.CreateOAuth2ApplicationOptions) - app, err := login.UpdateOAuth2Application(login.UpdateOAuth2ApplicationOptions{ + app, err := auth.UpdateOAuth2Application(auth.UpdateOAuth2ApplicationOptions{ Name: data.Name, UserID: ctx.User.ID, ID: appID, RedirectURIs: data.RedirectURIs, }) if err != nil { - if login.IsErrOauthClientIDInvalid(err) || login.IsErrOAuthApplicationNotFound(err) { + if auth.IsErrOauthClientIDInvalid(err) || auth.IsErrOAuthApplicationNotFound(err) { ctx.NotFound() } else { ctx.Error(http.StatusInternalServerError, "UpdateOauth2ApplicationByID", err) diff --git a/routers/web/admin/auths.go b/routers/web/admin/auths.go index b288273871f6..338a54c5dd26 100644 --- a/routers/web/admin/auths.go +++ b/routers/web/admin/auths.go @@ -13,7 +13,7 @@ import ( "strconv" "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/auth/pam" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" @@ -24,7 +24,7 @@ import ( auth_service "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/ldap" "code.gitea.io/gitea/services/auth/source/oauth2" - pamService "code.gitea.io/gitea/services/auth/source/pam" + pam_service "code.gitea.io/gitea/services/auth/source/pam" "code.gitea.io/gitea/services/auth/source/smtp" "code.gitea.io/gitea/services/auth/source/sspi" "code.gitea.io/gitea/services/forms" @@ -50,13 +50,13 @@ func Authentications(ctx *context.Context) { ctx.Data["PageIsAdminAuthentications"] = true var err error - ctx.Data["Sources"], err = login.Sources() + ctx.Data["Sources"], err = auth.Sources() if err != nil { - ctx.ServerError("login.Sources", err) + ctx.ServerError("auth.Sources", err) return } - ctx.Data["Total"] = login.CountSources() + ctx.Data["Total"] = auth.CountSources() ctx.HTML(http.StatusOK, tplAuths) } @@ -68,14 +68,14 @@ type dropdownItem struct { var ( authSources = func() []dropdownItem { items := []dropdownItem{ - {login.LDAP.String(), login.LDAP}, - {login.DLDAP.String(), login.DLDAP}, - {login.SMTP.String(), login.SMTP}, - {login.OAuth2.String(), login.OAuth2}, - {login.SSPI.String(), login.SSPI}, + {auth.LDAP.String(), auth.LDAP}, + {auth.DLDAP.String(), auth.DLDAP}, + {auth.SMTP.String(), auth.SMTP}, + {auth.OAuth2.String(), auth.OAuth2}, + {auth.SSPI.String(), auth.SSPI}, } if pam.Supported { - items = append(items, dropdownItem{login.Names[login.PAM], login.PAM}) + items = append(items, dropdownItem{auth.Names[auth.PAM], auth.PAM}) } return items }() @@ -93,8 +93,8 @@ func NewAuthSource(ctx *context.Context) { ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminAuthentications"] = true - ctx.Data["type"] = login.LDAP - ctx.Data["CurrentTypeName"] = login.Names[login.LDAP] + ctx.Data["type"] = auth.LDAP + ctx.Data["CurrentTypeName"] = auth.Names[auth.LDAP] ctx.Data["CurrentSecurityProtocol"] = ldap.SecurityProtocolNames[ldap.SecurityProtocolUnencrypted] ctx.Data["smtp_auth"] = "PLAIN" ctx.Data["is_active"] = true @@ -226,7 +226,7 @@ func NewAuthSourcePost(ctx *context.Context) { ctx.Data["PageIsAdmin"] = true ctx.Data["PageIsAdminAuthentications"] = true - ctx.Data["CurrentTypeName"] = login.Type(form.Type).String() + ctx.Data["CurrentTypeName"] = auth.Type(form.Type).String() ctx.Data["CurrentSecurityProtocol"] = ldap.SecurityProtocolNames[ldap.SecurityProtocol(form.SecurityProtocol)] ctx.Data["AuthSources"] = authSources ctx.Data["SecurityProtocols"] = securityProtocols @@ -242,29 +242,29 @@ func NewAuthSourcePost(ctx *context.Context) { hasTLS := false var config convert.Conversion - switch login.Type(form.Type) { - case login.LDAP, login.DLDAP: + switch auth.Type(form.Type) { + case auth.LDAP, auth.DLDAP: config = parseLDAPConfig(form) hasTLS = ldap.SecurityProtocol(form.SecurityProtocol) > ldap.SecurityProtocolUnencrypted - case login.SMTP: + case auth.SMTP: config = parseSMTPConfig(form) hasTLS = true - case login.PAM: - config = &pamService.Source{ + case auth.PAM: + config = &pam_service.Source{ ServiceName: form.PAMServiceName, EmailDomain: form.PAMEmailDomain, SkipLocalTwoFA: form.SkipLocalTwoFA, } - case login.OAuth2: + case auth.OAuth2: config = parseOAuth2Config(form) - case login.SSPI: + case auth.SSPI: var err error config, err = parseSSPIConfig(ctx, form) if err != nil { ctx.RenderWithErr(err.Error(), tplAuthNew, form) return } - existing, err := login.SourcesByType(login.SSPI) + existing, err := auth.SourcesByType(auth.SSPI) if err != nil || len(existing) > 0 { ctx.Data["Err_Type"] = true ctx.RenderWithErr(ctx.Tr("admin.auths.login_source_of_type_exist"), tplAuthNew, form) @@ -281,18 +281,18 @@ func NewAuthSourcePost(ctx *context.Context) { return } - if err := login.CreateSource(&login.Source{ - Type: login.Type(form.Type), + if err := auth.CreateSource(&auth.Source{ + Type: auth.Type(form.Type), Name: form.Name, IsActive: form.IsActive, IsSyncEnabled: form.IsSyncEnabled, Cfg: config, }); err != nil { - if login.IsErrSourceAlreadyExist(err) { + if auth.IsErrSourceAlreadyExist(err) { ctx.Data["Err_Name"] = true - ctx.RenderWithErr(ctx.Tr("admin.auths.login_source_exist", err.(login.ErrSourceAlreadyExist).Name), tplAuthNew, form) + ctx.RenderWithErr(ctx.Tr("admin.auths.login_source_exist", err.(auth.ErrSourceAlreadyExist).Name), tplAuthNew, form) } else { - ctx.ServerError("login.CreateSource", err) + ctx.ServerError("auth.CreateSource", err) } return } @@ -314,9 +314,9 @@ func EditAuthSource(ctx *context.Context) { oauth2providers := oauth2.GetOAuth2Providers() ctx.Data["OAuth2Providers"] = oauth2providers - source, err := login.GetSourceByID(ctx.ParamsInt64(":authid")) + source, err := auth.GetSourceByID(ctx.ParamsInt64(":authid")) if err != nil { - ctx.ServerError("login.GetSourceByID", err) + ctx.ServerError("auth.GetSourceByID", err) return } ctx.Data["Source"] = source @@ -349,9 +349,9 @@ func EditAuthSourcePost(ctx *context.Context) { oauth2providers := oauth2.GetOAuth2Providers() ctx.Data["OAuth2Providers"] = oauth2providers - source, err := login.GetSourceByID(ctx.ParamsInt64(":authid")) + source, err := auth.GetSourceByID(ctx.ParamsInt64(":authid")) if err != nil { - ctx.ServerError("login.GetSourceByID", err) + ctx.ServerError("auth.GetSourceByID", err) return } ctx.Data["Source"] = source @@ -363,19 +363,19 @@ func EditAuthSourcePost(ctx *context.Context) { } var config convert.Conversion - switch login.Type(form.Type) { - case login.LDAP, login.DLDAP: + switch auth.Type(form.Type) { + case auth.LDAP, auth.DLDAP: config = parseLDAPConfig(form) - case login.SMTP: + case auth.SMTP: config = parseSMTPConfig(form) - case login.PAM: - config = &pamService.Source{ + case auth.PAM: + config = &pam_service.Source{ ServiceName: form.PAMServiceName, EmailDomain: form.PAMEmailDomain, } - case login.OAuth2: + case auth.OAuth2: config = parseOAuth2Config(form) - case login.SSPI: + case auth.SSPI: config, err = parseSSPIConfig(ctx, form) if err != nil { ctx.RenderWithErr(err.Error(), tplAuthEdit, form) @@ -390,7 +390,7 @@ func EditAuthSourcePost(ctx *context.Context) { source.IsActive = form.IsActive source.IsSyncEnabled = form.IsSyncEnabled source.Cfg = config - if err := login.UpdateSource(source); err != nil { + if err := auth.UpdateSource(source); err != nil { if oauth2.IsErrOpenIDConnectInitialize(err) { ctx.Flash.Error(err.Error(), true) ctx.HTML(http.StatusOK, tplAuthEdit) @@ -407,17 +407,17 @@ func EditAuthSourcePost(ctx *context.Context) { // DeleteAuthSource response for deleting an auth source func DeleteAuthSource(ctx *context.Context) { - source, err := login.GetSourceByID(ctx.ParamsInt64(":authid")) + source, err := auth.GetSourceByID(ctx.ParamsInt64(":authid")) if err != nil { - ctx.ServerError("login.GetSourceByID", err) + ctx.ServerError("auth.GetSourceByID", err) return } - if err = auth_service.DeleteLoginSource(source); err != nil { - if login.IsErrSourceInUse(err) { + if err = auth_service.DeleteSource(source); err != nil { + if auth.IsErrSourceInUse(err) { ctx.Flash.Error(ctx.Tr("admin.auths.still_in_used")) } else { - ctx.Flash.Error(fmt.Sprintf("DeleteLoginSource: %v", err)) + ctx.Flash.Error(fmt.Sprintf("auth_service.DeleteSource: %v", err)) } ctx.JSON(http.StatusOK, map[string]interface{}{ "redirect": setting.AppSubURL + "/admin/auths/" + url.PathEscape(ctx.Params(":authid")), diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index 044efa009912..5cb25d8672e2 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -12,8 +12,8 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" @@ -81,9 +81,9 @@ func NewUser(ctx *context.Context) { ctx.Data["login_type"] = "0-0" - sources, err := login.Sources() + sources, err := auth.Sources() if err != nil { - ctx.ServerError("login.Sources", err) + ctx.ServerError("auth.Sources", err) return } ctx.Data["Sources"] = sources @@ -100,9 +100,9 @@ func NewUserPost(ctx *context.Context) { ctx.Data["PageIsAdminUsers"] = true ctx.Data["DefaultUserVisibilityMode"] = setting.Service.DefaultUserVisibilityMode - sources, err := login.Sources() + sources, err := auth.Sources() if err != nil { - ctx.ServerError("login.Sources", err) + ctx.ServerError("auth.Sources", err) return } ctx.Data["Sources"] = sources @@ -119,19 +119,19 @@ func NewUserPost(ctx *context.Context) { Email: form.Email, Passwd: form.Password, IsActive: true, - LoginType: login.Plain, + LoginType: auth.Plain, } if len(form.LoginType) > 0 { fields := strings.Split(form.LoginType, "-") if len(fields) == 2 { lType, _ := strconv.ParseInt(fields[0], 10, 0) - u.LoginType = login.Type(lType) + u.LoginType = auth.Type(lType) u.LoginSource, _ = strconv.ParseInt(fields[1], 10, 64) u.LoginName = form.LoginName } } - if u.LoginType == login.NoType || u.LoginType == login.Plain { + if u.LoginType == auth.NoType || u.LoginType == auth.Plain { if len(form.Password) < setting.MinPasswordLength { ctx.Data["Err_Password"] = true ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplUserNew, &form) @@ -201,26 +201,26 @@ func prepareUserInfo(ctx *context.Context) *user_model.User { ctx.Data["User"] = u if u.LoginSource > 0 { - ctx.Data["LoginSource"], err = login.GetSourceByID(u.LoginSource) + ctx.Data["LoginSource"], err = auth.GetSourceByID(u.LoginSource) if err != nil { - ctx.ServerError("login.GetSourceByID", err) + ctx.ServerError("auth.GetSourceByID", err) return nil } } else { - ctx.Data["LoginSource"] = &login.Source{} + ctx.Data["LoginSource"] = &auth.Source{} } - sources, err := login.Sources() + sources, err := auth.Sources() if err != nil { - ctx.ServerError("login.Sources", err) + ctx.ServerError("auth.Sources", err) return nil } ctx.Data["Sources"] = sources ctx.Data["TwoFactorEnabled"] = true - _, err = login.GetTwoFactorByUID(u.ID) + _, err = auth.GetTwoFactorByUID(u.ID) if err != nil { - if !login.IsErrTwoFactorNotEnrolled(err) { + if !auth.IsErrTwoFactorNotEnrolled(err) { ctx.ServerError("IsErrTwoFactorNotEnrolled", err) return nil } @@ -268,11 +268,11 @@ func EditUserPost(ctx *context.Context) { fields := strings.Split(form.LoginType, "-") if len(fields) == 2 { loginType, _ := strconv.ParseInt(fields[0], 10, 0) - loginSource, _ := strconv.ParseInt(fields[1], 10, 64) + authSource, _ := strconv.ParseInt(fields[1], 10, 64) - if u.LoginSource != loginSource { - u.LoginSource = loginSource - u.LoginType = login.Type(loginType) + if u.LoginSource != authSource { + u.LoginSource = authSource + u.LoginType = auth.Type(loginType) } } @@ -325,13 +325,13 @@ func EditUserPost(ctx *context.Context) { } if form.Reset2FA { - tf, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + tf, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { ctx.ServerError("GetTwoFactorByUID", err) return } - if err = login.DeleteTwoFactorByID(tf.ID, u.ID); err != nil { + if err = auth.DeleteTwoFactorByID(tf.ID, u.ID); err != nil { ctx.ServerError("DeleteTwoFactorByID", err) return } diff --git a/routers/web/auth/2fa.go b/routers/web/auth/2fa.go new file mode 100644 index 000000000000..c61922cd9da0 --- /dev/null +++ b/routers/web/auth/2fa.go @@ -0,0 +1,166 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "errors" + "net/http" + + "code.gitea.io/gitea/models/auth" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/externalaccount" + "code.gitea.io/gitea/services/forms" +) + +var ( + tplTwofa base.TplName = "user/auth/twofa" + tplTwofaScratch base.TplName = "user/auth/twofa_scratch" +) + +// TwoFactor shows the user a two-factor authentication page. +func TwoFactor(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("twofa") + + // Check auto-login. + if checkAutoLogin(ctx) { + return + } + + // Ensure user is in a 2FA session. + if ctx.Session.Get("twofaUid") == nil { + ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) + return + } + + ctx.HTML(http.StatusOK, tplTwofa) +} + +// TwoFactorPost validates a user's two-factor authentication token. +func TwoFactorPost(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.TwoFactorAuthForm) + ctx.Data["Title"] = ctx.Tr("twofa") + + // Ensure user is in a 2FA session. + idSess := ctx.Session.Get("twofaUid") + if idSess == nil { + ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) + return + } + + id := idSess.(int64) + twofa, err := auth.GetTwoFactorByUID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + // Validate the passcode with the stored TOTP secret. + ok, err := twofa.ValidateTOTP(form.Passcode) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + if ok && twofa.LastUsedPasscode != form.Passcode { + remember := ctx.Session.Get("twofaRemember").(bool) + u, err := user_model.GetUserByID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + if ctx.Session.Get("linkAccount") != nil { + err = externalaccount.LinkAccountFromStore(ctx.Session, u) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + } + + twofa.LastUsedPasscode = form.Passcode + if err = auth.UpdateTwoFactor(twofa); err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + handleSignIn(ctx, u, remember) + return + } + + ctx.RenderWithErr(ctx.Tr("auth.twofa_passcode_incorrect"), tplTwofa, forms.TwoFactorAuthForm{}) +} + +// TwoFactorScratch shows the scratch code form for two-factor authentication. +func TwoFactorScratch(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("twofa_scratch") + + // Check auto-login. + if checkAutoLogin(ctx) { + return + } + + // Ensure user is in a 2FA session. + if ctx.Session.Get("twofaUid") == nil { + ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) + return + } + + ctx.HTML(http.StatusOK, tplTwofaScratch) +} + +// TwoFactorScratchPost validates and invalidates a user's two-factor scratch token. +func TwoFactorScratchPost(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.TwoFactorScratchAuthForm) + ctx.Data["Title"] = ctx.Tr("twofa_scratch") + + // Ensure user is in a 2FA session. + idSess := ctx.Session.Get("twofaUid") + if idSess == nil { + ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) + return + } + + id := idSess.(int64) + twofa, err := auth.GetTwoFactorByUID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + // Validate the passcode with the stored TOTP secret. + if twofa.VerifyScratchToken(form.Token) { + // Invalidate the scratch token. + _, err = twofa.GenerateScratchToken() + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + if err = auth.UpdateTwoFactor(twofa); err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + remember := ctx.Session.Get("twofaRemember").(bool) + u, err := user_model.GetUserByID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + handleSignInFull(ctx, u, remember, false) + if ctx.Written() { + return + } + ctx.Flash.Info(ctx.Tr("auth.twofa_scratch_used")) + ctx.Redirect(setting.AppSubURL + "/user/settings/security") + return + } + + ctx.RenderWithErr(ctx.Tr("auth.twofa_scratch_token_incorrect"), tplTwofaScratch, forms.TwoFactorScratchAuthForm{}) +} diff --git a/routers/web/auth/auth.go b/routers/web/auth/auth.go new file mode 100644 index 000000000000..b9765abfb548 --- /dev/null +++ b/routers/web/auth/auth.go @@ -0,0 +1,795 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/eventsource" + "code.gitea.io/gitea/modules/hcaptcha" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/password" + "code.gitea.io/gitea/modules/recaptcha" + "code.gitea.io/gitea/modules/session" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/modules/web/middleware" + "code.gitea.io/gitea/routers/utils" + auth_service "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/auth/source/oauth2" + "code.gitea.io/gitea/services/externalaccount" + "code.gitea.io/gitea/services/forms" + "code.gitea.io/gitea/services/mailer" + + "github.com/markbates/goth" +) + +const ( + // tplSignIn template for sign in page + tplSignIn base.TplName = "user/auth/signin" + // tplSignUp template path for sign up page + tplSignUp base.TplName = "user/auth/signup" + // TplActivate template path for activate user + TplActivate base.TplName = "user/auth/activate" +) + +// AutoSignIn reads cookie and try to auto-login. +func AutoSignIn(ctx *context.Context) (bool, error) { + if !db.HasEngine { + return false, nil + } + + uname := ctx.GetCookie(setting.CookieUserName) + if len(uname) == 0 { + return false, nil + } + + isSucceed := false + defer func() { + if !isSucceed { + log.Trace("auto-login cookie cleared: %s", uname) + ctx.DeleteCookie(setting.CookieUserName) + ctx.DeleteCookie(setting.CookieRememberName) + } + }() + + u, err := user_model.GetUserByName(uname) + if err != nil { + if !user_model.IsErrUserNotExist(err) { + return false, fmt.Errorf("GetUserByName: %v", err) + } + return false, nil + } + + if val, ok := ctx.GetSuperSecureCookie( + base.EncodeMD5(u.Rands+u.Passwd), setting.CookieRememberName); !ok || val != u.Name { + return false, nil + } + + isSucceed = true + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + return false, fmt.Errorf("unable to RegenerateSession: Error: %w", err) + } + + // Set session IDs + if err := ctx.Session.Set("uid", u.ID); err != nil { + return false, err + } + if err := ctx.Session.Set("uname", u.Name); err != nil { + return false, err + } + if err := ctx.Session.Release(); err != nil { + return false, err + } + + if err := resetLocale(ctx, u); err != nil { + return false, err + } + + middleware.DeleteCSRFCookie(ctx.Resp) + return true, nil +} + +func resetLocale(ctx *context.Context, u *user_model.User) error { + // Language setting of the user overwrites the one previously set + // If the user does not have a locale set, we save the current one. + if len(u.Language) == 0 { + u.Language = ctx.Locale.Language() + if err := user_model.UpdateUserCols(db.DefaultContext, u, "language"); err != nil { + return err + } + } + + middleware.SetLocaleCookie(ctx.Resp, u.Language, 0) + + if ctx.Locale.Language() != u.Language { + ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) + } + + return nil +} + +func checkAutoLogin(ctx *context.Context) bool { + // Check auto-login + isSucceed, err := AutoSignIn(ctx) + if err != nil { + ctx.ServerError("AutoSignIn", err) + return true + } + + redirectTo := ctx.FormString("redirect_to") + if len(redirectTo) > 0 { + middleware.SetRedirectToCookie(ctx.Resp, redirectTo) + } else { + redirectTo = ctx.GetCookie("redirect_to") + } + + if isSucceed { + middleware.DeleteRedirectToCookie(ctx.Resp) + ctx.RedirectToFirst(redirectTo, setting.AppSubURL+string(setting.LandingPageURL)) + return true + } + + return false +} + +// SignIn render sign in page +func SignIn(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("sign_in") + + // Check auto-login + if checkAutoLogin(ctx) { + return + } + + orderedOAuth2Names, oauth2Providers, err := oauth2.GetActiveOAuth2Providers() + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names + ctx.Data["OAuth2Providers"] = oauth2Providers + ctx.Data["Title"] = ctx.Tr("sign_in") + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" + ctx.Data["PageIsSignIn"] = true + ctx.Data["PageIsLogin"] = true + ctx.Data["EnableSSPI"] = auth.IsSSPIEnabled() + + ctx.HTML(http.StatusOK, tplSignIn) +} + +// SignInPost response for sign in request +func SignInPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("sign_in") + + orderedOAuth2Names, oauth2Providers, err := oauth2.GetActiveOAuth2Providers() + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names + ctx.Data["OAuth2Providers"] = oauth2Providers + ctx.Data["Title"] = ctx.Tr("sign_in") + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" + ctx.Data["PageIsSignIn"] = true + ctx.Data["PageIsLogin"] = true + ctx.Data["EnableSSPI"] = auth.IsSSPIEnabled() + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplSignIn) + return + } + + form := web.GetForm(ctx).(*forms.SignInForm) + u, source, err := auth_service.UserSignIn(form.UserName, form.Password) + if err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplSignIn, &form) + log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) + } else if user_model.IsErrEmailAlreadyUsed(err) { + ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tplSignIn, &form) + log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) + } else if user_model.IsErrUserProhibitLogin(err) { + log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) + ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") + ctx.HTML(http.StatusOK, "user/auth/prohibit_login") + } else if user_model.IsErrUserInactive(err) { + if setting.Service.RegisterEmailConfirm { + ctx.Data["Title"] = ctx.Tr("auth.active_your_account") + ctx.HTML(http.StatusOK, TplActivate) + } else { + log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) + ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") + ctx.HTML(http.StatusOK, "user/auth/prohibit_login") + } + } else { + ctx.ServerError("UserSignIn", err) + } + return + } + + // Now handle 2FA: + + // First of all if the source can skip local two fa we're done + if skipper, ok := source.Cfg.(auth_service.LocalTwoFASkipper); ok && skipper.IsSkipLocalTwoFA() { + handleSignIn(ctx, u, form.Remember) + return + } + + // If this user is enrolled in 2FA TOTP, we can't sign the user in just yet. + // Instead, redirect them to the 2FA authentication page. + hasTOTPtwofa, err := auth.HasTwoFactorByUID(u.ID) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + // Check if the user has u2f registration + hasU2Ftwofa, err := auth.HasU2FRegistrationsByUID(u.ID) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + if !hasTOTPtwofa && !hasU2Ftwofa { + // No two factor auth configured we can sign in the user + handleSignIn(ctx, u, form.Remember) + return + } + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("UserSignIn: Unable to set regenerate session", err) + return + } + + // User will need to use 2FA TOTP or U2F, save data + if err := ctx.Session.Set("twofaUid", u.ID); err != nil { + ctx.ServerError("UserSignIn: Unable to set twofaUid in session", err) + return + } + + if err := ctx.Session.Set("twofaRemember", form.Remember); err != nil { + ctx.ServerError("UserSignIn: Unable to set twofaRemember in session", err) + return + } + + if hasTOTPtwofa { + // User will need to use U2F, save data + if err := ctx.Session.Set("totpEnrolled", u.ID); err != nil { + ctx.ServerError("UserSignIn: Unable to set u2fEnrolled in session", err) + return + } + } + + if err := ctx.Session.Release(); err != nil { + ctx.ServerError("UserSignIn: Unable to save session", err) + return + } + + // If we have U2F redirect there first + if hasU2Ftwofa { + ctx.Redirect(setting.AppSubURL + "/user/u2f") + return + } + + // Fallback to 2FA + ctx.Redirect(setting.AppSubURL + "/user/two_factor") +} + +// This handles the final part of the sign-in process of the user. +func handleSignIn(ctx *context.Context, u *user_model.User, remember bool) { + redirect := handleSignInFull(ctx, u, remember, true) + if ctx.Written() { + return + } + ctx.Redirect(redirect) +} + +func handleSignInFull(ctx *context.Context, u *user_model.User, remember bool, obeyRedirect bool) string { + if remember { + days := 86400 * setting.LogInRememberDays + ctx.SetCookie(setting.CookieUserName, u.Name, days) + ctx.SetSuperSecureCookie(base.EncodeMD5(u.Rands+u.Passwd), + setting.CookieRememberName, u.Name, days) + } + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("RegenerateSession", err) + return setting.AppSubURL + "/" + } + + // Delete the openid, 2fa and linkaccount data + _ = ctx.Session.Delete("openid_verified_uri") + _ = ctx.Session.Delete("openid_signin_remember") + _ = ctx.Session.Delete("openid_determined_email") + _ = ctx.Session.Delete("openid_determined_username") + _ = ctx.Session.Delete("twofaUid") + _ = ctx.Session.Delete("twofaRemember") + _ = ctx.Session.Delete("u2fChallenge") + _ = ctx.Session.Delete("linkAccount") + if err := ctx.Session.Set("uid", u.ID); err != nil { + log.Error("Error setting uid %d in session: %v", u.ID, err) + } + if err := ctx.Session.Set("uname", u.Name); err != nil { + log.Error("Error setting uname %s session: %v", u.Name, err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Unable to store session: %v", err) + } + + // Language setting of the user overwrites the one previously set + // If the user does not have a locale set, we save the current one. + if len(u.Language) == 0 { + u.Language = ctx.Locale.Language() + if err := user_model.UpdateUserCols(db.DefaultContext, u, "language"); err != nil { + ctx.ServerError("UpdateUserCols Language", fmt.Errorf("Error updating user language [user: %d, locale: %s]", u.ID, u.Language)) + return setting.AppSubURL + "/" + } + } + + middleware.SetLocaleCookie(ctx.Resp, u.Language, 0) + + if ctx.Locale.Language() != u.Language { + ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) + } + + // Clear whatever CSRF has right now, force to generate a new one + middleware.DeleteCSRFCookie(ctx.Resp) + + // Register last login + u.SetLastLogin() + if err := user_model.UpdateUserCols(db.DefaultContext, u, "last_login_unix"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return setting.AppSubURL + "/" + } + + if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 && !utils.IsExternalURL(redirectTo) { + middleware.DeleteRedirectToCookie(ctx.Resp) + if obeyRedirect { + ctx.RedirectToFirst(redirectTo) + } + return redirectTo + } + + if obeyRedirect { + ctx.Redirect(setting.AppSubURL + "/") + } + return setting.AppSubURL + "/" +} + +func getUserName(gothUser *goth.User) string { + switch setting.OAuth2Client.Username { + case setting.OAuth2UsernameEmail: + return strings.Split(gothUser.Email, "@")[0] + case setting.OAuth2UsernameNickname: + return gothUser.NickName + default: // OAuth2UsernameUserid + return gothUser.UserID + } +} + +// HandleSignOut resets the session and sets the cookies +func HandleSignOut(ctx *context.Context) { + _ = ctx.Session.Flush() + _ = ctx.Session.Destroy(ctx.Resp, ctx.Req) + ctx.DeleteCookie(setting.CookieUserName) + ctx.DeleteCookie(setting.CookieRememberName) + middleware.DeleteCSRFCookie(ctx.Resp) + middleware.DeleteLocaleCookie(ctx.Resp) + middleware.DeleteRedirectToCookie(ctx.Resp) +} + +// SignOut sign out from login status +func SignOut(ctx *context.Context) { + if ctx.User != nil { + eventsource.GetManager().SendMessageBlocking(ctx.User.ID, &eventsource.Event{ + Name: "logout", + Data: ctx.Session.ID(), + }) + } + HandleSignOut(ctx) + ctx.Redirect(setting.AppSubURL + "/") +} + +// SignUp render the register page +func SignUp(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("sign_up") + + ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/sign_up" + + ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha + ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL + ctx.Data["Captcha"] = context.GetImageCaptcha() + ctx.Data["CaptchaType"] = setting.Service.CaptchaType + ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey + ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey + ctx.Data["PageIsSignUp"] = true + + //Show Disabled Registration message if DisableRegistration or AllowOnlyExternalRegistration options are true + ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration + + ctx.HTML(http.StatusOK, tplSignUp) +} + +// SignUpPost response for sign up information submission +func SignUpPost(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.RegisterForm) + ctx.Data["Title"] = ctx.Tr("sign_up") + + ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/sign_up" + + ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha + ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL + ctx.Data["Captcha"] = context.GetImageCaptcha() + ctx.Data["CaptchaType"] = setting.Service.CaptchaType + ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey + ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey + ctx.Data["PageIsSignUp"] = true + + //Permission denied if DisableRegistration or AllowOnlyExternalRegistration options are true + if setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration { + ctx.Error(http.StatusForbidden) + return + } + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplSignUp) + return + } + + if setting.Service.EnableCaptcha { + var valid bool + var err error + switch setting.Service.CaptchaType { + case setting.ImageCaptcha: + valid = context.GetImageCaptcha().VerifyReq(ctx.Req) + case setting.ReCaptcha: + valid, err = recaptcha.Verify(ctx, form.GRecaptchaResponse) + case setting.HCaptcha: + valid, err = hcaptcha.Verify(ctx, form.HcaptchaResponse) + default: + ctx.ServerError("Unknown Captcha Type", fmt.Errorf("Unknown Captcha Type: %s", setting.Service.CaptchaType)) + return + } + if err != nil { + log.Debug("%s", err.Error()) + } + + if !valid { + ctx.Data["Err_Captcha"] = true + ctx.RenderWithErr(ctx.Tr("form.captcha_incorrect"), tplSignUp, &form) + return + } + } + + if !form.IsEmailDomainAllowed() { + ctx.RenderWithErr(ctx.Tr("auth.email_domain_blacklisted"), tplSignUp, &form) + return + } + + if form.Password != form.Retype { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplSignUp, &form) + return + } + if len(form.Password) < setting.MinPasswordLength { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplSignUp, &form) + return + } + if !password.IsComplexEnough(form.Password) { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(password.BuildComplexityError(ctx), tplSignUp, &form) + return + } + pwned, err := password.IsPwned(ctx, form.Password) + if pwned { + errMsg := ctx.Tr("auth.password_pwned") + if err != nil { + log.Error(err.Error()) + errMsg = ctx.Tr("auth.password_pwned_err") + } + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(errMsg, tplSignUp, &form) + return + } + + u := &user_model.User{ + Name: form.UserName, + Email: form.Email, + Passwd: form.Password, + IsActive: !(setting.Service.RegisterEmailConfirm || setting.Service.RegisterManualConfirm), + IsRestricted: setting.Service.DefaultUserIsRestricted, + } + + if !createAndHandleCreatedUser(ctx, tplSignUp, form, u, nil, false) { + // error already handled + return + } + + ctx.Flash.Success(ctx.Tr("auth.sign_up_successful")) + handleSignIn(ctx, u, false) +} + +// createAndHandleCreatedUser calls createUserInContext and +// then handleUserCreated. +func createAndHandleCreatedUser(ctx *context.Context, tpl base.TplName, form interface{}, u *user_model.User, gothUser *goth.User, allowLink bool) bool { + if !createUserInContext(ctx, tpl, form, u, gothUser, allowLink) { + return false + } + return handleUserCreated(ctx, u, gothUser) +} + +// createUserInContext creates a user and handles errors within a given context. +// Optionally a template can be specified. +func createUserInContext(ctx *context.Context, tpl base.TplName, form interface{}, u *user_model.User, gothUser *goth.User, allowLink bool) (ok bool) { + if err := user_model.CreateUser(u); err != nil { + if allowLink && (user_model.IsErrUserAlreadyExist(err) || user_model.IsErrEmailAlreadyUsed(err)) { + if setting.OAuth2Client.AccountLinking == setting.OAuth2AccountLinkingAuto { + var user *user_model.User + user = &user_model.User{Name: u.Name} + hasUser, err := user_model.GetUser(user) + if !hasUser || err != nil { + user = &user_model.User{Email: u.Email} + hasUser, err = user_model.GetUser(user) + if !hasUser || err != nil { + ctx.ServerError("UserLinkAccount", err) + return + } + } + + // TODO: probably we should respect 'remember' user's choice... + linkAccount(ctx, user, *gothUser, true) + return // user is already created here, all redirects are handled + } else if setting.OAuth2Client.AccountLinking == setting.OAuth2AccountLinkingLogin { + showLinkingLogin(ctx, *gothUser) + return // user will be created only after linking login + } + } + + // handle error without template + if len(tpl) == 0 { + ctx.ServerError("CreateUser", err) + return + } + + // handle error with template + switch { + case user_model.IsErrUserAlreadyExist(err): + ctx.Data["Err_UserName"] = true + ctx.RenderWithErr(ctx.Tr("form.username_been_taken"), tpl, form) + case user_model.IsErrEmailAlreadyUsed(err): + ctx.Data["Err_Email"] = true + ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tpl, form) + case user_model.IsErrEmailInvalid(err): + ctx.Data["Err_Email"] = true + ctx.RenderWithErr(ctx.Tr("form.email_invalid"), tpl, form) + case db.IsErrNameReserved(err): + ctx.Data["Err_UserName"] = true + ctx.RenderWithErr(ctx.Tr("user.form.name_reserved", err.(db.ErrNameReserved).Name), tpl, form) + case db.IsErrNamePatternNotAllowed(err): + ctx.Data["Err_UserName"] = true + ctx.RenderWithErr(ctx.Tr("user.form.name_pattern_not_allowed", err.(db.ErrNamePatternNotAllowed).Pattern), tpl, form) + case db.IsErrNameCharsNotAllowed(err): + ctx.Data["Err_UserName"] = true + ctx.RenderWithErr(ctx.Tr("user.form.name_chars_not_allowed", err.(db.ErrNameCharsNotAllowed).Name), tpl, form) + default: + ctx.ServerError("CreateUser", err) + } + return + } + log.Trace("Account created: %s", u.Name) + return true +} + +// handleUserCreated does additional steps after a new user is created. +// It auto-sets admin for the only user, updates the optional external user and +// sends a confirmation email if required. +func handleUserCreated(ctx *context.Context, u *user_model.User, gothUser *goth.User) (ok bool) { + // Auto-set admin for the only user. + if user_model.CountUsers() == 1 { + u.IsAdmin = true + u.IsActive = true + u.SetLastLogin() + if err := user_model.UpdateUserCols(db.DefaultContext, u, "is_admin", "is_active", "last_login_unix"); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + } + + // update external user information + if gothUser != nil { + if err := externalaccount.UpdateExternalUser(u, *gothUser); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + } + + // Send confirmation email + if !u.IsActive && u.ID > 1 { + mailer.SendActivateAccountMail(ctx.Locale, u) + + ctx.Data["IsSendRegisterMail"] = true + ctx.Data["Email"] = u.Email + ctx.Data["ActiveCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language()) + ctx.HTML(http.StatusOK, TplActivate) + + if err := ctx.Cache.Put("MailResendLimit_"+u.LowerName, u.LowerName, 180); err != nil { + log.Error("Set cache(MailResendLimit) fail: %v", err) + } + return + } + + return true +} + +// Activate render activate user page +func Activate(ctx *context.Context) { + code := ctx.FormString("code") + + if len(code) == 0 { + ctx.Data["IsActivatePage"] = true + if ctx.User == nil || ctx.User.IsActive { + ctx.NotFound("invalid user", nil) + return + } + // Resend confirmation email. + if setting.Service.RegisterEmailConfirm { + if ctx.Cache.IsExist("MailResendLimit_" + ctx.User.LowerName) { + ctx.Data["ResendLimited"] = true + } else { + ctx.Data["ActiveCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language()) + mailer.SendActivateAccountMail(ctx.Locale, ctx.User) + + if err := ctx.Cache.Put("MailResendLimit_"+ctx.User.LowerName, ctx.User.LowerName, 180); err != nil { + log.Error("Set cache(MailResendLimit) fail: %v", err) + } + } + } else { + ctx.Data["ServiceNotEnabled"] = true + } + ctx.HTML(http.StatusOK, TplActivate) + return + } + + user := user_model.VerifyUserActiveCode(code) + // if code is wrong + if user == nil { + ctx.Data["IsActivateFailed"] = true + ctx.HTML(http.StatusOK, TplActivate) + return + } + + // if account is local account, verify password + if user.LoginSource == 0 { + ctx.Data["Code"] = code + ctx.Data["NeedsPassword"] = true + ctx.HTML(http.StatusOK, TplActivate) + return + } + + handleAccountActivation(ctx, user) +} + +// ActivatePost handles account activation with password check +func ActivatePost(ctx *context.Context) { + code := ctx.FormString("code") + if len(code) == 0 { + ctx.Redirect(setting.AppSubURL + "/user/activate") + return + } + + user := user_model.VerifyUserActiveCode(code) + // if code is wrong + if user == nil { + ctx.Data["IsActivateFailed"] = true + ctx.HTML(http.StatusOK, TplActivate) + return + } + + // if account is local account, verify password + if user.LoginSource == 0 { + password := ctx.FormString("password") + if len(password) == 0 { + ctx.Data["Code"] = code + ctx.Data["NeedsPassword"] = true + ctx.HTML(http.StatusOK, TplActivate) + return + } + if !user.ValidatePassword(password) { + ctx.Data["IsActivateFailed"] = true + ctx.HTML(http.StatusOK, TplActivate) + return + } + } + + handleAccountActivation(ctx, user) +} + +func handleAccountActivation(ctx *context.Context, user *user_model.User) { + user.IsActive = true + var err error + if user.Rands, err = user_model.GetUserSalt(); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + if err := user_model.UpdateUserCols(db.DefaultContext, user, "is_active", "rands"); err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.NotFound("UpdateUserCols", err) + } else { + ctx.ServerError("UpdateUser", err) + } + return + } + + if err := user_model.ActivateUserEmail(user.ID, user.Email, true); err != nil { + log.Error("Unable to activate email for user: %-v with email: %s: %v", user, user.Email, err) + ctx.ServerError("ActivateUserEmail", err) + return + } + + log.Trace("User activated: %s", user.Name) + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + log.Error("Unable to regenerate session for user: %-v with email: %s: %v", user, user.Email, err) + ctx.ServerError("ActivateUserEmail", err) + return + } + + if err := ctx.Session.Set("uid", user.ID); err != nil { + log.Error("Error setting uid in session[%s]: %v", ctx.Session.ID(), err) + } + if err := ctx.Session.Set("uname", user.Name); err != nil { + log.Error("Error setting uname in session[%s]: %v", ctx.Session.ID(), err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Error storing session[%s]: %v", ctx.Session.ID(), err) + } + + if err := resetLocale(ctx, user); err != nil { + ctx.ServerError("resetLocale", err) + return + } + + ctx.Flash.Success(ctx.Tr("auth.account_activated")) + ctx.Redirect(setting.AppSubURL + "/") +} + +// ActivateEmail render the activate email page +func ActivateEmail(ctx *context.Context) { + code := ctx.FormString("code") + emailStr := ctx.FormString("email") + + // Verify code. + if email := user_model.VerifyActiveEmailCode(code, emailStr); email != nil { + if err := user_model.ActivateEmail(email); err != nil { + ctx.ServerError("ActivateEmail", err) + } + + log.Trace("Email activated: %s", email.Email) + ctx.Flash.Success(ctx.Tr("settings.add_email_success")) + + if u, err := user_model.GetUserByID(email.UID); err != nil { + log.Warn("GetUserByID: %d", email.UID) + } else { + // Allow user to validate more emails + _ = ctx.Cache.Delete("MailResendLimit_" + u.LowerName) + } + } + + // FIXME: e-mail verification does not require the user to be logged in, + // so this could be redirecting to the login page. + // Should users be logged in automatically here? (consider 2FA requirements, etc.) + ctx.Redirect(setting.AppSubURL + "/user/settings/account") +} diff --git a/routers/web/auth/linkaccount.go b/routers/web/auth/linkaccount.go new file mode 100644 index 000000000000..9d5a6eb3f815 --- /dev/null +++ b/routers/web/auth/linkaccount.go @@ -0,0 +1,300 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "errors" + "fmt" + "net/http" + "strings" + + "code.gitea.io/gitea/models/auth" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/hcaptcha" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/recaptcha" + "code.gitea.io/gitea/modules/session" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + auth_service "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/services/externalaccount" + "code.gitea.io/gitea/services/forms" + + "github.com/markbates/goth" +) + +var ( + tplLinkAccount base.TplName = "user/auth/link_account" +) + +// LinkAccount shows the page where the user can decide to login or create a new account +func LinkAccount(ctx *context.Context) { + ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration + ctx.Data["Title"] = ctx.Tr("link_account") + ctx.Data["LinkAccountMode"] = true + ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha + ctx.Data["Captcha"] = context.GetImageCaptcha() + ctx.Data["CaptchaType"] = setting.Service.CaptchaType + ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL + ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey + ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey + ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration + ctx.Data["AllowOnlyInternalRegistration"] = setting.Service.AllowOnlyInternalRegistration + ctx.Data["ShowRegistrationButton"] = false + + // use this to set the right link into the signIn and signUp templates in the link_account template + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" + ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" + + gothUser := ctx.Session.Get("linkAccountGothUser") + if gothUser == nil { + ctx.ServerError("UserSignIn", errors.New("not in LinkAccount session")) + return + } + + gu, _ := gothUser.(goth.User) + uname := getUserName(&gu) + email := gu.Email + ctx.Data["user_name"] = uname + ctx.Data["email"] = email + + if len(email) != 0 { + u, err := user_model.GetUserByEmail(email) + if err != nil && !user_model.IsErrUserNotExist(err) { + ctx.ServerError("UserSignIn", err) + return + } + if u != nil { + ctx.Data["user_exists"] = true + } + } else if len(uname) != 0 { + u, err := user_model.GetUserByName(uname) + if err != nil && !user_model.IsErrUserNotExist(err) { + ctx.ServerError("UserSignIn", err) + return + } + if u != nil { + ctx.Data["user_exists"] = true + } + } + + ctx.HTML(http.StatusOK, tplLinkAccount) +} + +// LinkAccountPostSignIn handle the coupling of external account with another account using signIn +func LinkAccountPostSignIn(ctx *context.Context) { + signInForm := web.GetForm(ctx).(*forms.SignInForm) + ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration + ctx.Data["Title"] = ctx.Tr("link_account") + ctx.Data["LinkAccountMode"] = true + ctx.Data["LinkAccountModeSignIn"] = true + ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha + ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL + ctx.Data["Captcha"] = context.GetImageCaptcha() + ctx.Data["CaptchaType"] = setting.Service.CaptchaType + ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey + ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey + ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration + ctx.Data["ShowRegistrationButton"] = false + + // use this to set the right link into the signIn and signUp templates in the link_account template + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" + ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" + + gothUser := ctx.Session.Get("linkAccountGothUser") + if gothUser == nil { + ctx.ServerError("UserSignIn", errors.New("not in LinkAccount session")) + return + } + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplLinkAccount) + return + } + + u, _, err := auth_service.UserSignIn(signInForm.UserName, signInForm.Password) + if err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.Data["user_exists"] = true + ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplLinkAccount, &signInForm) + } else { + ctx.ServerError("UserLinkAccount", err) + } + return + } + + linkAccount(ctx, u, gothUser.(goth.User), signInForm.Remember) +} + +func linkAccount(ctx *context.Context, u *user_model.User, gothUser goth.User, remember bool) { + updateAvatarIfNeed(gothUser.AvatarURL, u) + + // If this user is enrolled in 2FA, we can't sign the user in just yet. + // Instead, redirect them to the 2FA authentication page. + // We deliberately ignore the skip local 2fa setting here because we are linking to a previous user here + _, err := auth.GetTwoFactorByUID(u.ID) + if err != nil { + if !auth.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserLinkAccount", err) + return + } + + err = externalaccount.LinkAccountToUser(u, gothUser) + if err != nil { + ctx.ServerError("UserLinkAccount", err) + return + } + + handleSignIn(ctx, u, remember) + return + } + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("RegenerateSession", err) + return + } + + // User needs to use 2FA, save data and redirect to 2FA page. + if err := ctx.Session.Set("twofaUid", u.ID); err != nil { + log.Error("Error setting twofaUid in session: %v", err) + } + if err := ctx.Session.Set("twofaRemember", remember); err != nil { + log.Error("Error setting twofaRemember in session: %v", err) + } + if err := ctx.Session.Set("linkAccount", true); err != nil { + log.Error("Error setting linkAccount in session: %v", err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Error storing session: %v", err) + } + + // If U2F is enrolled -> Redirect to U2F instead + regs, err := auth.GetU2FRegistrationsByUID(u.ID) + if err == nil && len(regs) > 0 { + ctx.Redirect(setting.AppSubURL + "/user/u2f") + return + } + + ctx.Redirect(setting.AppSubURL + "/user/two_factor") +} + +// LinkAccountPostRegister handle the creation of a new account for an external account using signUp +func LinkAccountPostRegister(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.RegisterForm) + // TODO Make insecure passwords optional for local accounts also, + // once email-based Second-Factor Auth is available + ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration + ctx.Data["Title"] = ctx.Tr("link_account") + ctx.Data["LinkAccountMode"] = true + ctx.Data["LinkAccountModeRegister"] = true + ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha + ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL + ctx.Data["Captcha"] = context.GetImageCaptcha() + ctx.Data["CaptchaType"] = setting.Service.CaptchaType + ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey + ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey + ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration + ctx.Data["ShowRegistrationButton"] = false + + // use this to set the right link into the signIn and signUp templates in the link_account template + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" + ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" + + gothUserInterface := ctx.Session.Get("linkAccountGothUser") + if gothUserInterface == nil { + ctx.ServerError("UserSignUp", errors.New("not in LinkAccount session")) + return + } + gothUser, ok := gothUserInterface.(goth.User) + if !ok { + ctx.ServerError("UserSignUp", fmt.Errorf("session linkAccountGothUser type is %t but not goth.User", gothUserInterface)) + return + } + + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplLinkAccount) + return + } + + if setting.Service.DisableRegistration || setting.Service.AllowOnlyInternalRegistration { + ctx.Error(http.StatusForbidden) + return + } + + if setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha { + var valid bool + var err error + switch setting.Service.CaptchaType { + case setting.ImageCaptcha: + valid = context.GetImageCaptcha().VerifyReq(ctx.Req) + case setting.ReCaptcha: + valid, err = recaptcha.Verify(ctx, form.GRecaptchaResponse) + case setting.HCaptcha: + valid, err = hcaptcha.Verify(ctx, form.HcaptchaResponse) + default: + ctx.ServerError("Unknown Captcha Type", fmt.Errorf("Unknown Captcha Type: %s", setting.Service.CaptchaType)) + return + } + if err != nil { + log.Debug("%s", err.Error()) + } + + if !valid { + ctx.Data["Err_Captcha"] = true + ctx.RenderWithErr(ctx.Tr("form.captcha_incorrect"), tplLinkAccount, &form) + return + } + } + + if !form.IsEmailDomainAllowed() { + ctx.RenderWithErr(ctx.Tr("auth.email_domain_blacklisted"), tplLinkAccount, &form) + return + } + + if setting.Service.AllowOnlyExternalRegistration || !setting.Service.RequireExternalRegistrationPassword { + // In user_model.User an empty password is classed as not set, so we set form.Password to empty. + // Eventually the database should be changed to indicate "Second Factor"-enabled accounts + // (accounts that do not introduce the security vulnerabilities of a password). + // If a user decides to circumvent second-factor security, and purposefully create a password, + // they can still do so using the "Recover Account" option. + form.Password = "" + } else { + if (len(strings.TrimSpace(form.Password)) > 0 || len(strings.TrimSpace(form.Retype)) > 0) && form.Password != form.Retype { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplLinkAccount, &form) + return + } + if len(strings.TrimSpace(form.Password)) > 0 && len(form.Password) < setting.MinPasswordLength { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplLinkAccount, &form) + return + } + } + + authSource, err := auth.GetActiveOAuth2SourceByName(gothUser.Provider) + if err != nil { + ctx.ServerError("CreateUser", err) + return + } + + u := &user_model.User{ + Name: form.UserName, + Email: form.Email, + Passwd: form.Password, + IsActive: !(setting.Service.RegisterEmailConfirm || setting.Service.RegisterManualConfirm), + LoginType: auth.OAuth2, + LoginSource: authSource.ID, + LoginName: gothUser.UserID, + } + + if !createAndHandleCreatedUser(ctx, tplLinkAccount, form, u, &gothUser, false) { + // error already handled + return + } + + handleSignIn(ctx, u, false) +} diff --git a/routers/web/auth/main_test.go b/routers/web/auth/main_test.go new file mode 100644 index 000000000000..2b16f3c4059b --- /dev/null +++ b/routers/web/auth/main_test.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/unittest" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m, filepath.Join("..", "..", "..")) +} diff --git a/routers/web/user/oauth.go b/routers/web/auth/oauth.go similarity index 65% rename from routers/web/user/oauth.go rename to routers/web/auth/oauth.go index 166420221d26..9b22773d2f0d 100644 --- a/routers/web/user/oauth.go +++ b/routers/web/auth/oauth.go @@ -2,32 +2,40 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package user +package auth import ( "encoding/base64" + "errors" "fmt" "html" + "io" "net/http" "net/url" "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/session" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/web" - "code.gitea.io/gitea/services/auth" + "code.gitea.io/gitea/modules/web/middleware" + auth_service "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" + "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/forms" + user_service "code.gitea.io/gitea/services/user" "gitea.com/go-chi/binding" "github.com/golang-jwt/jwt" + "github.com/markbates/goth" ) const ( @@ -117,7 +125,7 @@ type AccessTokenResponse struct { IDToken string `json:"id_token,omitempty"` } -func newAccessTokenResponse(grant *login.OAuth2Grant, serverKey, clientKey oauth2.JWTSigningKey) (*AccessTokenResponse, *AccessTokenError) { +func newAccessTokenResponse(grant *auth.OAuth2Grant, serverKey, clientKey oauth2.JWTSigningKey) (*AccessTokenResponse, *AccessTokenError) { if setting.OAuth2.InvalidateRefreshTokens { if err := grant.IncreaseCounter(); err != nil { return nil, &AccessTokenError{ @@ -164,7 +172,7 @@ func newAccessTokenResponse(grant *login.OAuth2Grant, serverKey, clientKey oauth // generate OpenID Connect id_token signedIDToken := "" if grant.ScopeContains("openid") { - app, err := login.GetOAuth2ApplicationByID(grant.ApplicationID) + app, err := auth.GetOAuth2ApplicationByID(grant.ApplicationID) if err != nil { return nil, &AccessTokenError{ ErrorCode: AccessTokenErrorCodeInvalidRequest, @@ -249,7 +257,7 @@ type userInfoResponse struct { // InfoOAuth manages request for userinfo endpoint func InfoOAuth(ctx *context.Context) { - if ctx.User == nil || ctx.Data["AuthedMethod"] != (&auth.OAuth2{}).Name() { + if ctx.User == nil || ctx.Data["AuthedMethod"] != (&auth_service.OAuth2{}).Name() { ctx.Resp.Header().Set("WWW-Authenticate", `Bearer realm=""`) ctx.PlainText(http.StatusUnauthorized, "no valid authorization") return @@ -315,9 +323,9 @@ func IntrospectOAuth(ctx *context.Context) { token, err := oauth2.ParseToken(form.Token, oauth2.DefaultSigningKey) if err == nil { if token.Valid() == nil { - grant, err := login.GetOAuth2GrantByID(token.GrantID) + grant, err := auth.GetOAuth2GrantByID(token.GrantID) if err == nil && grant != nil { - app, err := login.GetOAuth2ApplicationByID(grant.ApplicationID) + app, err := auth.GetOAuth2ApplicationByID(grant.ApplicationID) if err == nil && app != nil { response.Active = true response.Scope = grant.Scope @@ -346,9 +354,9 @@ func AuthorizeOAuth(ctx *context.Context) { return } - app, err := login.GetOAuth2ApplicationByClientID(form.ClientID) + app, err := auth.GetOAuth2ApplicationByClientID(form.ClientID) if err != nil { - if login.IsErrOauthClientIDInvalid(err) { + if auth.IsErrOauthClientIDInvalid(err) { handleAuthorizeError(ctx, AuthorizeError{ ErrorCode: ErrorCodeUnauthorizedClient, ErrorDescription: "Client ID not registered", @@ -492,7 +500,7 @@ func GrantApplicationOAuth(ctx *context.Context) { ctx.Error(http.StatusBadRequest) return } - app, err := login.GetOAuth2ApplicationByClientID(form.ClientID) + app, err := auth.GetOAuth2ApplicationByClientID(form.ClientID) if err != nil { ctx.ServerError("GetOAuth2ApplicationByClientID", err) return @@ -630,7 +638,7 @@ func handleRefreshToken(ctx *context.Context, form forms.AccessTokenForm, server return } // get grant before increasing counter - grant, err := login.GetOAuth2GrantByID(token.GrantID) + grant, err := auth.GetOAuth2GrantByID(token.GrantID) if err != nil || grant == nil { handleAccessTokenError(ctx, AccessTokenError{ ErrorCode: AccessTokenErrorCodeInvalidGrant, @@ -657,7 +665,7 @@ func handleRefreshToken(ctx *context.Context, form forms.AccessTokenForm, server } func handleAuthorizationCode(ctx *context.Context, form forms.AccessTokenForm, serverKey, clientKey oauth2.JWTSigningKey) { - app, err := login.GetOAuth2ApplicationByClientID(form.ClientID) + app, err := auth.GetOAuth2ApplicationByClientID(form.ClientID) if err != nil { handleAccessTokenError(ctx, AccessTokenError{ ErrorCode: AccessTokenErrorCodeInvalidClient, @@ -679,7 +687,7 @@ func handleAuthorizationCode(ctx *context.Context, form forms.AccessTokenForm, s }) return } - authorizationCode, err := login.GetOAuth2AuthorizationByCode(form.Code) + authorizationCode, err := auth.GetOAuth2AuthorizationByCode(form.Code) if err != nil || authorizationCode == nil { handleAccessTokenError(ctx, AccessTokenError{ ErrorCode: AccessTokenErrorCodeUnauthorizedClient, @@ -750,3 +758,367 @@ func handleAuthorizeError(ctx *context.Context, authErr AuthorizeError, redirect redirect.RawQuery = q.Encode() ctx.Redirect(redirect.String(), 302) } + +// SignInOAuth handles the OAuth2 login buttons +func SignInOAuth(ctx *context.Context) { + provider := ctx.Params(":provider") + + authSource, err := auth.GetActiveOAuth2SourceByName(provider) + if err != nil { + ctx.ServerError("SignIn", err) + return + } + + // try to do a direct callback flow, so we don't authenticate the user again but use the valid accesstoken to get the user + user, gothUser, err := oAuth2UserLoginCallback(authSource, ctx.Req, ctx.Resp) + if err == nil && user != nil { + // we got the user without going through the whole OAuth2 authentication flow again + handleOAuth2SignIn(ctx, authSource, user, gothUser) + return + } + + if err = authSource.Cfg.(*oauth2.Source).Callout(ctx.Req, ctx.Resp); err != nil { + if strings.Contains(err.Error(), "no provider for ") { + if err = oauth2.ResetOAuth2(); err != nil { + ctx.ServerError("SignIn", err) + return + } + if err = authSource.Cfg.(*oauth2.Source).Callout(ctx.Req, ctx.Resp); err != nil { + ctx.ServerError("SignIn", err) + } + return + } + ctx.ServerError("SignIn", err) + } + // redirect is done in oauth2.Auth +} + +// SignInOAuthCallback handles the callback from the given provider +func SignInOAuthCallback(ctx *context.Context) { + provider := ctx.Params(":provider") + + // first look if the provider is still active + authSource, err := auth.GetActiveOAuth2SourceByName(provider) + if err != nil { + ctx.ServerError("SignIn", err) + return + } + + if authSource == nil { + ctx.ServerError("SignIn", errors.New("No valid provider found, check configured callback url in provider")) + return + } + + u, gothUser, err := oAuth2UserLoginCallback(authSource, ctx.Req, ctx.Resp) + + if err != nil { + if user_model.IsErrUserProhibitLogin(err) { + uplerr := err.(*user_model.ErrUserProhibitLogin) + log.Info("Failed authentication attempt for %s from %s: %v", uplerr.Name, ctx.RemoteAddr(), err) + ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") + ctx.HTML(http.StatusOK, "user/auth/prohibit_login") + return + } + ctx.ServerError("UserSignIn", err) + return + } + + if u == nil { + if !setting.Service.AllowOnlyInternalRegistration && setting.OAuth2Client.EnableAutoRegistration { + // create new user with details from oauth2 provider + var missingFields []string + if gothUser.UserID == "" { + missingFields = append(missingFields, "sub") + } + if gothUser.Email == "" { + missingFields = append(missingFields, "email") + } + if setting.OAuth2Client.Username == setting.OAuth2UsernameNickname && gothUser.NickName == "" { + missingFields = append(missingFields, "nickname") + } + if len(missingFields) > 0 { + log.Error("OAuth2 Provider %s returned empty or missing fields: %s", authSource.Name, missingFields) + if authSource.IsOAuth2() && authSource.Cfg.(*oauth2.Source).Provider == "openidConnect" { + log.Error("You may need to change the 'OPENID_CONNECT_SCOPES' setting to request all required fields") + } + err = fmt.Errorf("OAuth2 Provider %s returned empty or missing fields: %s", authSource.Name, missingFields) + ctx.ServerError("CreateUser", err) + return + } + u = &user_model.User{ + Name: getUserName(&gothUser), + FullName: gothUser.Name, + Email: gothUser.Email, + IsActive: !setting.OAuth2Client.RegisterEmailConfirm, + LoginType: auth.OAuth2, + LoginSource: authSource.ID, + LoginName: gothUser.UserID, + IsRestricted: setting.Service.DefaultUserIsRestricted, + } + + setUserGroupClaims(authSource, u, &gothUser) + + if !createAndHandleCreatedUser(ctx, base.TplName(""), nil, u, &gothUser, setting.OAuth2Client.AccountLinking != setting.OAuth2AccountLinkingDisabled) { + // error already handled + return + } + } else { + // no existing user is found, request attach or new account + showLinkingLogin(ctx, gothUser) + return + } + } + + handleOAuth2SignIn(ctx, authSource, u, gothUser) +} + +func claimValueToStringSlice(claimValue interface{}) []string { + var groups []string + + switch rawGroup := claimValue.(type) { + case []string: + groups = rawGroup + default: + str := fmt.Sprintf("%s", rawGroup) + groups = strings.Split(str, ",") + } + return groups +} + +func setUserGroupClaims(loginSource *auth.Source, u *user_model.User, gothUser *goth.User) bool { + source := loginSource.Cfg.(*oauth2.Source) + if source.GroupClaimName == "" || (source.AdminGroup == "" && source.RestrictedGroup == "") { + return false + } + + groupClaims, has := gothUser.RawData[source.GroupClaimName] + if !has { + return false + } + + groups := claimValueToStringSlice(groupClaims) + + wasAdmin, wasRestricted := u.IsAdmin, u.IsRestricted + + if source.AdminGroup != "" { + u.IsAdmin = false + } + if source.RestrictedGroup != "" { + u.IsRestricted = false + } + + for _, g := range groups { + if source.AdminGroup != "" && g == source.AdminGroup { + u.IsAdmin = true + } else if source.RestrictedGroup != "" && g == source.RestrictedGroup { + u.IsRestricted = true + } + } + + return wasAdmin != u.IsAdmin || wasRestricted != u.IsRestricted +} + +func showLinkingLogin(ctx *context.Context, gothUser goth.User) { + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("RegenerateSession", err) + return + } + + if err := ctx.Session.Set("linkAccountGothUser", gothUser); err != nil { + log.Error("Error setting linkAccountGothUser in session: %v", err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Error storing session: %v", err) + } + ctx.Redirect(setting.AppSubURL + "/user/link_account") +} + +func updateAvatarIfNeed(url string, u *user_model.User) { + if setting.OAuth2Client.UpdateAvatar && len(url) > 0 { + resp, err := http.Get(url) + if err == nil { + defer func() { + _ = resp.Body.Close() + }() + } + // ignore any error + if err == nil && resp.StatusCode == http.StatusOK { + data, err := io.ReadAll(io.LimitReader(resp.Body, setting.Avatar.MaxFileSize+1)) + if err == nil && int64(len(data)) <= setting.Avatar.MaxFileSize { + _ = user_service.UploadAvatar(u, data) + } + } + } +} + +func handleOAuth2SignIn(ctx *context.Context, source *auth.Source, u *user_model.User, gothUser goth.User) { + updateAvatarIfNeed(gothUser.AvatarURL, u) + + needs2FA := false + if !source.Cfg.(*oauth2.Source).SkipLocalTwoFA { + _, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { + ctx.ServerError("UserSignIn", err) + return + } + needs2FA = err == nil + } + + // If this user is enrolled in 2FA and this source doesn't override it, + // we can't sign the user in just yet. Instead, redirect them to the 2FA authentication page. + if !needs2FA { + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("RegenerateSession", err) + return + } + + if err := ctx.Session.Set("uid", u.ID); err != nil { + log.Error("Error setting uid in session: %v", err) + } + if err := ctx.Session.Set("uname", u.Name); err != nil { + log.Error("Error setting uname in session: %v", err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Error storing session: %v", err) + } + + // Clear whatever CSRF has right now, force to generate a new one + middleware.DeleteCSRFCookie(ctx.Resp) + + // Register last login + u.SetLastLogin() + + // Update GroupClaims + changed := setUserGroupClaims(source, u, &gothUser) + cols := []string{"last_login_unix"} + if changed { + cols = append(cols, "is_admin", "is_restricted") + } + + if err := user_model.UpdateUserCols(db.DefaultContext, u, cols...); err != nil { + ctx.ServerError("UpdateUserCols", err) + return + } + + // update external user information + if err := externalaccount.UpdateExternalUser(u, gothUser); err != nil { + log.Error("UpdateExternalUser failed: %v", err) + } + + if err := resetLocale(ctx, u); err != nil { + ctx.ServerError("resetLocale", err) + return + } + + if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { + middleware.DeleteRedirectToCookie(ctx.Resp) + ctx.RedirectToFirst(redirectTo) + return + } + + ctx.Redirect(setting.AppSubURL + "/") + return + } + + changed := setUserGroupClaims(source, u, &gothUser) + if changed { + if err := user_model.UpdateUserCols(db.DefaultContext, u, "is_admin", "is_restricted"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return + } + } + + if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { + ctx.ServerError("RegenerateSession", err) + return + } + + // User needs to use 2FA, save data and redirect to 2FA page. + if err := ctx.Session.Set("twofaUid", u.ID); err != nil { + log.Error("Error setting twofaUid in session: %v", err) + } + if err := ctx.Session.Set("twofaRemember", false); err != nil { + log.Error("Error setting twofaRemember in session: %v", err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Error storing session: %v", err) + } + + // If U2F is enrolled -> Redirect to U2F instead + regs, err := auth.GetU2FRegistrationsByUID(u.ID) + if err == nil && len(regs) > 0 { + ctx.Redirect(setting.AppSubURL + "/user/u2f") + return + } + + ctx.Redirect(setting.AppSubURL + "/user/two_factor") +} + +// OAuth2UserLoginCallback attempts to handle the callback from the OAuth2 provider and if successful +// login the user +func oAuth2UserLoginCallback(authSource *auth.Source, request *http.Request, response http.ResponseWriter) (*user_model.User, goth.User, error) { + oauth2Source := authSource.Cfg.(*oauth2.Source) + + gothUser, err := oauth2Source.Callback(request, response) + if err != nil { + if err.Error() == "securecookie: the value is too long" || strings.Contains(err.Error(), "Data too long") { + log.Error("OAuth2 Provider %s returned too long a token. Current max: %d. Either increase the [OAuth2] MAX_TOKEN_LENGTH or reduce the information returned from the OAuth2 provider", authSource.Name, setting.OAuth2.MaxTokenLength) + err = fmt.Errorf("OAuth2 Provider %s returned too long a token. Current max: %d. Either increase the [OAuth2] MAX_TOKEN_LENGTH or reduce the information returned from the OAuth2 provider", authSource.Name, setting.OAuth2.MaxTokenLength) + } + return nil, goth.User{}, err + } + + if oauth2Source.RequiredClaimName != "" { + claimInterface, has := gothUser.RawData[oauth2Source.RequiredClaimName] + if !has { + return nil, goth.User{}, user_model.ErrUserProhibitLogin{Name: gothUser.UserID} + } + + if oauth2Source.RequiredClaimValue != "" { + groups := claimValueToStringSlice(claimInterface) + found := false + for _, group := range groups { + if group == oauth2Source.RequiredClaimValue { + found = true + break + } + } + if !found { + return nil, goth.User{}, user_model.ErrUserProhibitLogin{Name: gothUser.UserID} + } + } + } + + user := &user_model.User{ + LoginName: gothUser.UserID, + LoginType: auth.OAuth2, + LoginSource: authSource.ID, + } + + hasUser, err := user_model.GetUser(user) + if err != nil { + return nil, goth.User{}, err + } + + if hasUser { + return user, gothUser, nil + } + + // search in external linked users + externalLoginUser := &user_model.ExternalLoginUser{ + ExternalID: gothUser.UserID, + LoginSourceID: authSource.ID, + } + hasUser, err = user_model.GetExternalLogin(externalLoginUser) + if err != nil { + return nil, goth.User{}, err + } + if hasUser { + user, err = user_model.GetUserByID(externalLoginUser.UserID) + return user, gothUser, err + } + + // no user found to login + return nil, gothUser, nil + +} diff --git a/routers/web/user/oauth_test.go b/routers/web/auth/oauth_test.go similarity index 90% rename from routers/web/user/oauth_test.go rename to routers/web/auth/oauth_test.go index e26a42b2fcf4..c652d901f3ef 100644 --- a/routers/web/user/oauth_test.go +++ b/routers/web/auth/oauth_test.go @@ -2,12 +2,12 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package user +package auth import ( "testing" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/services/auth/source/oauth2" @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" ) -func createAndParseToken(t *testing.T, grant *login.OAuth2Grant) *oauth2.OIDCToken { +func createAndParseToken(t *testing.T, grant *auth.OAuth2Grant) *oauth2.OIDCToken { signingKey, err := oauth2.CreateJWTSigningKey("HS256", make([]byte, 32)) assert.NoError(t, err) assert.NotNil(t, signingKey) @@ -43,7 +43,7 @@ func createAndParseToken(t *testing.T, grant *login.OAuth2Grant) *oauth2.OIDCTok func TestNewAccessTokenResponse_OIDCToken(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - grants, err := login.GetOAuth2GrantsByUserID(3) + grants, err := auth.GetOAuth2GrantsByUserID(3) assert.NoError(t, err) assert.Len(t, grants, 1) @@ -59,7 +59,7 @@ func TestNewAccessTokenResponse_OIDCToken(t *testing.T) { assert.False(t, oidcToken.EmailVerified) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 5}).(*user_model.User) - grants, err = login.GetOAuth2GrantsByUserID(user.ID) + grants, err = auth.GetOAuth2GrantsByUserID(user.ID) assert.NoError(t, err) assert.Len(t, grants, 1) diff --git a/routers/web/user/auth_openid.go b/routers/web/auth/openid.go similarity index 99% rename from routers/web/user/auth_openid.go rename to routers/web/auth/openid.go index 55dd8f8d09a8..4395641795fd 100644 --- a/routers/web/user/auth_openid.go +++ b/routers/web/auth/openid.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package user +package auth import ( "fmt" diff --git a/routers/web/auth/password.go b/routers/web/auth/password.go new file mode 100644 index 000000000000..65d5c55976ee --- /dev/null +++ b/routers/web/auth/password.go @@ -0,0 +1,346 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "errors" + "net/http" + + "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/password" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/modules/web/middleware" + "code.gitea.io/gitea/routers/utils" + "code.gitea.io/gitea/services/forms" + "code.gitea.io/gitea/services/mailer" +) + +var ( + // tplMustChangePassword template for updating a user's password + tplMustChangePassword base.TplName = "user/auth/change_passwd" + tplForgotPassword base.TplName = "user/auth/forgot_passwd" + tplResetPassword base.TplName = "user/auth/reset_passwd" +) + +// ForgotPasswd render the forget password page +func ForgotPasswd(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("auth.forgot_password_title") + + if setting.MailService == nil { + log.Warn(ctx.Tr("auth.disable_forgot_password_mail_admin")) + ctx.Data["IsResetDisable"] = true + ctx.HTML(http.StatusOK, tplForgotPassword) + return + } + + ctx.Data["Email"] = ctx.FormString("email") + + ctx.Data["IsResetRequest"] = true + ctx.HTML(http.StatusOK, tplForgotPassword) +} + +// ForgotPasswdPost response for forget password request +func ForgotPasswdPost(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("auth.forgot_password_title") + + if setting.MailService == nil { + ctx.NotFound("ForgotPasswdPost", nil) + return + } + ctx.Data["IsResetRequest"] = true + + email := ctx.FormString("email") + ctx.Data["Email"] = email + + u, err := user_model.GetUserByEmail(email) + if err != nil { + if user_model.IsErrUserNotExist(err) { + ctx.Data["ResetPwdCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, ctx.Locale.Language()) + ctx.Data["IsResetSent"] = true + ctx.HTML(http.StatusOK, tplForgotPassword) + return + } + + ctx.ServerError("user.ResetPasswd(check existence)", err) + return + } + + if !u.IsLocal() && !u.IsOAuth2() { + ctx.Data["Err_Email"] = true + ctx.RenderWithErr(ctx.Tr("auth.non_local_account"), tplForgotPassword, nil) + return + } + + if ctx.Cache.IsExist("MailResendLimit_" + u.LowerName) { + ctx.Data["ResendLimited"] = true + ctx.HTML(http.StatusOK, tplForgotPassword) + return + } + + mailer.SendResetPasswordMail(u) + + if err = ctx.Cache.Put("MailResendLimit_"+u.LowerName, u.LowerName, 180); err != nil { + log.Error("Set cache(MailResendLimit) fail: %v", err) + } + + ctx.Data["ResetPwdCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, ctx.Locale.Language()) + ctx.Data["IsResetSent"] = true + ctx.HTML(http.StatusOK, tplForgotPassword) +} + +func commonResetPassword(ctx *context.Context) (*user_model.User, *auth.TwoFactor) { + code := ctx.FormString("code") + + ctx.Data["Title"] = ctx.Tr("auth.reset_password") + ctx.Data["Code"] = code + + if nil != ctx.User { + ctx.Data["user_signed_in"] = true + } + + if len(code) == 0 { + ctx.Flash.Error(ctx.Tr("auth.invalid_code")) + return nil, nil + } + + // Fail early, don't frustrate the user + u := user_model.VerifyUserActiveCode(code) + if u == nil { + ctx.Flash.Error(ctx.Tr("auth.invalid_code")) + return nil, nil + } + + twofa, err := auth.GetTwoFactorByUID(u.ID) + if err != nil { + if !auth.IsErrTwoFactorNotEnrolled(err) { + ctx.Error(http.StatusInternalServerError, "CommonResetPassword", err.Error()) + return nil, nil + } + } else { + ctx.Data["has_two_factor"] = true + ctx.Data["scratch_code"] = ctx.FormBool("scratch_code") + } + + // Show the user that they are affecting the account that they intended to + ctx.Data["user_email"] = u.Email + + if nil != ctx.User && u.ID != ctx.User.ID { + ctx.Flash.Error(ctx.Tr("auth.reset_password_wrong_user", ctx.User.Email, u.Email)) + return nil, nil + } + + return u, twofa +} + +// ResetPasswd render the account recovery page +func ResetPasswd(ctx *context.Context) { + ctx.Data["IsResetForm"] = true + + commonResetPassword(ctx) + if ctx.Written() { + return + } + + ctx.HTML(http.StatusOK, tplResetPassword) +} + +// ResetPasswdPost response from account recovery request +func ResetPasswdPost(ctx *context.Context) { + u, twofa := commonResetPassword(ctx) + if ctx.Written() { + return + } + + if u == nil { + // Flash error has been set + ctx.HTML(http.StatusOK, tplResetPassword) + return + } + + // Validate password length. + passwd := ctx.FormString("password") + if len(passwd) < setting.MinPasswordLength { + ctx.Data["IsResetForm"] = true + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplResetPassword, nil) + return + } else if !password.IsComplexEnough(passwd) { + ctx.Data["IsResetForm"] = true + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(password.BuildComplexityError(ctx), tplResetPassword, nil) + return + } else if pwned, err := password.IsPwned(ctx, passwd); pwned || err != nil { + errMsg := ctx.Tr("auth.password_pwned") + if err != nil { + log.Error(err.Error()) + errMsg = ctx.Tr("auth.password_pwned_err") + } + ctx.Data["IsResetForm"] = true + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(errMsg, tplResetPassword, nil) + return + } + + // Handle two-factor + regenerateScratchToken := false + if twofa != nil { + if ctx.FormBool("scratch_code") { + if !twofa.VerifyScratchToken(ctx.FormString("token")) { + ctx.Data["IsResetForm"] = true + ctx.Data["Err_Token"] = true + ctx.RenderWithErr(ctx.Tr("auth.twofa_scratch_token_incorrect"), tplResetPassword, nil) + return + } + regenerateScratchToken = true + } else { + passcode := ctx.FormString("passcode") + ok, err := twofa.ValidateTOTP(passcode) + if err != nil { + ctx.Error(http.StatusInternalServerError, "ValidateTOTP", err.Error()) + return + } + if !ok || twofa.LastUsedPasscode == passcode { + ctx.Data["IsResetForm"] = true + ctx.Data["Err_Passcode"] = true + ctx.RenderWithErr(ctx.Tr("auth.twofa_passcode_incorrect"), tplResetPassword, nil) + return + } + + twofa.LastUsedPasscode = passcode + if err = auth.UpdateTwoFactor(twofa); err != nil { + ctx.ServerError("ResetPasswdPost: UpdateTwoFactor", err) + return + } + } + } + var err error + if u.Rands, err = user_model.GetUserSalt(); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + if err = u.SetPassword(passwd); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + u.MustChangePassword = false + if err := user_model.UpdateUserCols(db.DefaultContext, u, "must_change_password", "passwd", "passwd_hash_algo", "rands", "salt"); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + + log.Trace("User password reset: %s", u.Name) + ctx.Data["IsResetFailed"] = true + remember := len(ctx.FormString("remember")) != 0 + + if regenerateScratchToken { + // Invalidate the scratch token. + _, err = twofa.GenerateScratchToken() + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + if err = auth.UpdateTwoFactor(twofa); err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + handleSignInFull(ctx, u, remember, false) + if ctx.Written() { + return + } + ctx.Flash.Info(ctx.Tr("auth.twofa_scratch_used")) + ctx.Redirect(setting.AppSubURL + "/user/settings/security") + return + } + + handleSignIn(ctx, u, remember) +} + +// MustChangePassword renders the page to change a user's password +func MustChangePassword(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("auth.must_change_password") + ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/settings/change_password" + ctx.Data["MustChangePassword"] = true + ctx.HTML(http.StatusOK, tplMustChangePassword) +} + +// MustChangePasswordPost response for updating a user's password after his/her +// account was created by an admin +func MustChangePasswordPost(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.MustChangePasswordForm) + ctx.Data["Title"] = ctx.Tr("auth.must_change_password") + ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/settings/change_password" + if ctx.HasError() { + ctx.HTML(http.StatusOK, tplMustChangePassword) + return + } + u := ctx.User + // Make sure only requests for users who are eligible to change their password via + // this method passes through + if !u.MustChangePassword { + ctx.ServerError("MustUpdatePassword", errors.New("cannot update password.. Please visit the settings page")) + return + } + + if form.Password != form.Retype { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplMustChangePassword, &form) + return + } + + if len(form.Password) < setting.MinPasswordLength { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplMustChangePassword, &form) + return + } + + if !password.IsComplexEnough(form.Password) { + ctx.Data["Err_Password"] = true + ctx.RenderWithErr(password.BuildComplexityError(ctx), tplMustChangePassword, &form) + return + } + pwned, err := password.IsPwned(ctx, form.Password) + if pwned { + ctx.Data["Err_Password"] = true + errMsg := ctx.Tr("auth.password_pwned") + if err != nil { + log.Error(err.Error()) + errMsg = ctx.Tr("auth.password_pwned_err") + } + ctx.RenderWithErr(errMsg, tplMustChangePassword, &form) + return + } + + if err = u.SetPassword(form.Password); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + + u.MustChangePassword = false + + if err := user_model.UpdateUserCols(db.DefaultContext, u, "must_change_password", "passwd", "passwd_hash_algo", "salt"); err != nil { + ctx.ServerError("UpdateUser", err) + return + } + + ctx.Flash.Success(ctx.Tr("settings.change_password_success")) + + log.Trace("User updated password: %s", u.Name) + + if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 && !utils.IsExternalURL(redirectTo) { + middleware.DeleteRedirectToCookie(ctx.Resp) + ctx.RedirectToFirst(redirectTo) + return + } + + ctx.Redirect(setting.AppSubURL + "/") +} diff --git a/routers/web/auth/u2f.go b/routers/web/auth/u2f.go new file mode 100644 index 000000000000..915671cd1e3d --- /dev/null +++ b/routers/web/auth/u2f.go @@ -0,0 +1,136 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package auth + +import ( + "errors" + "net/http" + + "code.gitea.io/gitea/models/auth" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/externalaccount" + + "github.com/tstranex/u2f" +) + +var tplU2F base.TplName = "user/auth/u2f" + +// U2F shows the U2F login page +func U2F(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("twofa") + ctx.Data["RequireU2F"] = true + // Check auto-login. + if checkAutoLogin(ctx) { + return + } + + // Ensure user is in a 2FA session. + if ctx.Session.Get("twofaUid") == nil { + ctx.ServerError("UserSignIn", errors.New("not in U2F session")) + return + } + + // See whether TOTP is also available. + if ctx.Session.Get("totpEnrolled") != nil { + ctx.Data["TOTPEnrolled"] = true + } + + ctx.HTML(http.StatusOK, tplU2F) +} + +// U2FChallenge submits a sign challenge to the browser +func U2FChallenge(ctx *context.Context) { + // Ensure user is in a U2F session. + idSess := ctx.Session.Get("twofaUid") + if idSess == nil { + ctx.ServerError("UserSignIn", errors.New("not in U2F session")) + return + } + id := idSess.(int64) + regs, err := auth.GetU2FRegistrationsByUID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + if len(regs) == 0 { + ctx.ServerError("UserSignIn", errors.New("no device registered")) + return + } + challenge, err := u2f.NewChallenge(setting.U2F.AppID, setting.U2F.TrustedFacets) + if err != nil { + ctx.ServerError("u2f.NewChallenge", err) + return + } + if err := ctx.Session.Set("u2fChallenge", challenge); err != nil { + ctx.ServerError("UserSignIn: unable to set u2fChallenge in session", err) + return + } + if err := ctx.Session.Release(); err != nil { + ctx.ServerError("UserSignIn: unable to store session", err) + } + + ctx.JSON(http.StatusOK, challenge.SignRequest(regs.ToRegistrations())) +} + +// U2FSign authenticates the user by signResp +func U2FSign(ctx *context.Context) { + signResp := web.GetForm(ctx).(*u2f.SignResponse) + challSess := ctx.Session.Get("u2fChallenge") + idSess := ctx.Session.Get("twofaUid") + if challSess == nil || idSess == nil { + ctx.ServerError("UserSignIn", errors.New("not in U2F session")) + return + } + challenge := challSess.(*u2f.Challenge) + id := idSess.(int64) + regs, err := auth.GetU2FRegistrationsByUID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + for _, reg := range regs { + r, err := reg.Parse() + if err != nil { + log.Error("parsing u2f registration: %v", err) + continue + } + newCounter, authErr := r.Authenticate(*signResp, *challenge, reg.Counter) + if authErr == nil { + reg.Counter = newCounter + user, err := user_model.GetUserByID(id) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + remember := ctx.Session.Get("twofaRemember").(bool) + if err := reg.UpdateCounter(); err != nil { + ctx.ServerError("UserSignIn", err) + return + } + + if ctx.Session.Get("linkAccount") != nil { + if err := externalaccount.LinkAccountFromStore(ctx.Session, user); err != nil { + ctx.ServerError("UserSignIn", err) + return + } + } + redirect := handleSignInFull(ctx, user, remember, false) + if ctx.Written() { + return + } + if redirect == "" { + redirect = setting.AppSubURL + "/" + } + ctx.PlainText(http.StatusOK, redirect) + return + } + } + ctx.Error(http.StatusUnauthorized) +} diff --git a/routers/web/events/events.go b/routers/web/events/events.go index 974aa755d16c..41f52375c3f5 100644 --- a/routers/web/events/events.go +++ b/routers/web/events/events.go @@ -17,7 +17,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/routers/web/user" + "code.gitea.io/gitea/routers/web/auth" ) // Events listens for events @@ -133,7 +133,7 @@ loop: }).WriteTo(ctx.Resp) ctx.Resp.Flush() go unregister() - user.HandleSignOut(ctx) + auth.HandleSignOut(ctx) break loop } // Replace the event - we don't want to expose the session ID to the user diff --git a/routers/web/home.go b/routers/web/home.go index f50197691ffd..ed81d84c7fd6 100644 --- a/routers/web/home.go +++ b/routers/web/home.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web/middleware" + "code.gitea.io/gitea/routers/web/auth" "code.gitea.io/gitea/routers/web/user" ) @@ -26,7 +27,7 @@ func Home(ctx *context.Context) { if ctx.IsSigned { if !ctx.User.IsActive && setting.Service.RegisterEmailConfirm { ctx.Data["Title"] = ctx.Tr("auth.active_your_account") - ctx.HTML(http.StatusOK, user.TplActivate) + ctx.HTML(http.StatusOK, auth.TplActivate) } else if !ctx.User.IsActive || ctx.User.ProhibitLogin { log.Info("Failed authentication attempt for %s from %s", ctx.User.Name, ctx.RemoteAddr()) ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 64e617dc4a3c..6ebf6a789abc 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -20,8 +20,8 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" @@ -179,12 +179,12 @@ func httpBase(ctx *context.Context) (h *serviceHandler) { } if ctx.IsBasicAuth && ctx.Data["IsApiToken"] != true { - _, err = login.GetTwoFactorByUID(ctx.User.ID) + _, err = auth.GetTwoFactorByUID(ctx.User.ID) if err == nil { // TODO: This response should be changed to "invalid credentials" for security reasons once the expectation behind it (creating an app token to authenticate) is properly documented ctx.PlainText(http.StatusUnauthorized, "Users with two-factor authentication enabled cannot perform HTTP/HTTPS operations via plain username and password. Please create and use a personal access token on the user settings page") return - } else if !login.IsErrTwoFactorNotEnrolled(err) { + } else if !auth.IsErrTwoFactorNotEnrolled(err) { ctx.ServerError("IsErrTwoFactorNotEnrolled", err) return } diff --git a/routers/web/user/auth.go b/routers/web/user/auth.go deleted file mode 100644 index 356a907859b0..000000000000 --- a/routers/web/user/auth.go +++ /dev/null @@ -1,2006 +0,0 @@ -// Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2018 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package user - -import ( - "errors" - "fmt" - "io" - "net/http" - "strings" - - "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/eventsource" - "code.gitea.io/gitea/modules/hcaptcha" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/password" - "code.gitea.io/gitea/modules/recaptcha" - "code.gitea.io/gitea/modules/session" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/web" - "code.gitea.io/gitea/modules/web/middleware" - "code.gitea.io/gitea/routers/utils" - "code.gitea.io/gitea/services/auth" - "code.gitea.io/gitea/services/auth/source/oauth2" - "code.gitea.io/gitea/services/externalaccount" - "code.gitea.io/gitea/services/forms" - "code.gitea.io/gitea/services/mailer" - user_service "code.gitea.io/gitea/services/user" - - "github.com/markbates/goth" - "github.com/tstranex/u2f" -) - -const ( - // tplMustChangePassword template for updating a user's password - tplMustChangePassword = "user/auth/change_passwd" - // tplSignIn template for sign in page - tplSignIn base.TplName = "user/auth/signin" - // tplSignUp template path for sign up page - tplSignUp base.TplName = "user/auth/signup" - // TplActivate template path for activate user - TplActivate base.TplName = "user/auth/activate" - tplForgotPassword base.TplName = "user/auth/forgot_passwd" - tplResetPassword base.TplName = "user/auth/reset_passwd" - tplTwofa base.TplName = "user/auth/twofa" - tplTwofaScratch base.TplName = "user/auth/twofa_scratch" - tplLinkAccount base.TplName = "user/auth/link_account" - tplU2F base.TplName = "user/auth/u2f" -) - -// AutoSignIn reads cookie and try to auto-login. -func AutoSignIn(ctx *context.Context) (bool, error) { - if !db.HasEngine { - return false, nil - } - - uname := ctx.GetCookie(setting.CookieUserName) - if len(uname) == 0 { - return false, nil - } - - isSucceed := false - defer func() { - if !isSucceed { - log.Trace("auto-login cookie cleared: %s", uname) - ctx.DeleteCookie(setting.CookieUserName) - ctx.DeleteCookie(setting.CookieRememberName) - } - }() - - u, err := user_model.GetUserByName(uname) - if err != nil { - if !user_model.IsErrUserNotExist(err) { - return false, fmt.Errorf("GetUserByName: %v", err) - } - return false, nil - } - - if val, ok := ctx.GetSuperSecureCookie( - base.EncodeMD5(u.Rands+u.Passwd), setting.CookieRememberName); !ok || val != u.Name { - return false, nil - } - - isSucceed = true - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - return false, fmt.Errorf("unable to RegenerateSession: Error: %w", err) - } - - // Set session IDs - if err := ctx.Session.Set("uid", u.ID); err != nil { - return false, err - } - if err := ctx.Session.Set("uname", u.Name); err != nil { - return false, err - } - if err := ctx.Session.Release(); err != nil { - return false, err - } - - if err := resetLocale(ctx, u); err != nil { - return false, err - } - - middleware.DeleteCSRFCookie(ctx.Resp) - return true, nil -} - -func resetLocale(ctx *context.Context, u *user_model.User) error { - // Language setting of the user overwrites the one previously set - // If the user does not have a locale set, we save the current one. - if len(u.Language) == 0 { - u.Language = ctx.Locale.Language() - if err := user_model.UpdateUserCols(db.DefaultContext, u, "language"); err != nil { - return err - } - } - - middleware.SetLocaleCookie(ctx.Resp, u.Language, 0) - - if ctx.Locale.Language() != u.Language { - ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) - } - - return nil -} - -func checkAutoLogin(ctx *context.Context) bool { - // Check auto-login. - isSucceed, err := AutoSignIn(ctx) - if err != nil { - ctx.ServerError("AutoSignIn", err) - return true - } - - redirectTo := ctx.FormString("redirect_to") - if len(redirectTo) > 0 { - middleware.SetRedirectToCookie(ctx.Resp, redirectTo) - } else { - redirectTo = ctx.GetCookie("redirect_to") - } - - if isSucceed { - middleware.DeleteRedirectToCookie(ctx.Resp) - ctx.RedirectToFirst(redirectTo, setting.AppSubURL+string(setting.LandingPageURL)) - return true - } - - return false -} - -// SignIn render sign in page -func SignIn(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("sign_in") - - // Check auto-login. - if checkAutoLogin(ctx) { - return - } - - orderedOAuth2Names, oauth2Providers, err := oauth2.GetActiveOAuth2Providers() - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names - ctx.Data["OAuth2Providers"] = oauth2Providers - ctx.Data["Title"] = ctx.Tr("sign_in") - ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" - ctx.Data["PageIsSignIn"] = true - ctx.Data["PageIsLogin"] = true - ctx.Data["EnableSSPI"] = login.IsSSPIEnabled() - - ctx.HTML(http.StatusOK, tplSignIn) -} - -// SignInPost response for sign in request -func SignInPost(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("sign_in") - - orderedOAuth2Names, oauth2Providers, err := oauth2.GetActiveOAuth2Providers() - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names - ctx.Data["OAuth2Providers"] = oauth2Providers - ctx.Data["Title"] = ctx.Tr("sign_in") - ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" - ctx.Data["PageIsSignIn"] = true - ctx.Data["PageIsLogin"] = true - ctx.Data["EnableSSPI"] = login.IsSSPIEnabled() - - if ctx.HasError() { - ctx.HTML(http.StatusOK, tplSignIn) - return - } - - form := web.GetForm(ctx).(*forms.SignInForm) - u, source, err := auth.UserSignIn(form.UserName, form.Password) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplSignIn, &form) - log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) - } else if user_model.IsErrEmailAlreadyUsed(err) { - ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tplSignIn, &form) - log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) - } else if user_model.IsErrUserProhibitLogin(err) { - log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) - ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") - ctx.HTML(http.StatusOK, "user/auth/prohibit_login") - } else if user_model.IsErrUserInactive(err) { - if setting.Service.RegisterEmailConfirm { - ctx.Data["Title"] = ctx.Tr("auth.active_your_account") - ctx.HTML(http.StatusOK, TplActivate) - } else { - log.Info("Failed authentication attempt for %s from %s: %v", form.UserName, ctx.RemoteAddr(), err) - ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") - ctx.HTML(http.StatusOK, "user/auth/prohibit_login") - } - } else { - ctx.ServerError("UserSignIn", err) - } - return - } - - // Now handle 2FA: - - // First of all if the source can skip local two fa we're done - if skipper, ok := source.Cfg.(auth.LocalTwoFASkipper); ok && skipper.IsSkipLocalTwoFA() { - handleSignIn(ctx, u, form.Remember) - return - } - - // If this user is enrolled in 2FA TOTP, we can't sign the user in just yet. - // Instead, redirect them to the 2FA authentication page. - hasTOTPtwofa, err := login.HasTwoFactorByUID(u.ID) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - // Check if the user has u2f registration - hasU2Ftwofa, err := login.HasU2FRegistrationsByUID(u.ID) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - if !hasTOTPtwofa && !hasU2Ftwofa { - // No two factor auth configured we can sign in the user - handleSignIn(ctx, u, form.Remember) - return - } - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("UserSignIn: Unable to set regenerate session", err) - return - } - - // User will need to use 2FA TOTP or U2F, save data - if err := ctx.Session.Set("twofaUid", u.ID); err != nil { - ctx.ServerError("UserSignIn: Unable to set twofaUid in session", err) - return - } - - if err := ctx.Session.Set("twofaRemember", form.Remember); err != nil { - ctx.ServerError("UserSignIn: Unable to set twofaRemember in session", err) - return - } - - if hasTOTPtwofa { - // User will need to use U2F, save data - if err := ctx.Session.Set("totpEnrolled", u.ID); err != nil { - ctx.ServerError("UserSignIn: Unable to set u2fEnrolled in session", err) - return - } - } - - if err := ctx.Session.Release(); err != nil { - ctx.ServerError("UserSignIn: Unable to save session", err) - return - } - - // If we have U2F redirect there first - if hasU2Ftwofa { - ctx.Redirect(setting.AppSubURL + "/user/u2f") - return - } - - // Fallback to 2FA - ctx.Redirect(setting.AppSubURL + "/user/two_factor") -} - -// TwoFactor shows the user a two-factor authentication page. -func TwoFactor(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("twofa") - - // Check auto-login. - if checkAutoLogin(ctx) { - return - } - - // Ensure user is in a 2FA session. - if ctx.Session.Get("twofaUid") == nil { - ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) - return - } - - ctx.HTML(http.StatusOK, tplTwofa) -} - -// TwoFactorPost validates a user's two-factor authentication token. -func TwoFactorPost(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.TwoFactorAuthForm) - ctx.Data["Title"] = ctx.Tr("twofa") - - // Ensure user is in a 2FA session. - idSess := ctx.Session.Get("twofaUid") - if idSess == nil { - ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) - return - } - - id := idSess.(int64) - twofa, err := login.GetTwoFactorByUID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - // Validate the passcode with the stored TOTP secret. - ok, err := twofa.ValidateTOTP(form.Passcode) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - if ok && twofa.LastUsedPasscode != form.Passcode { - remember := ctx.Session.Get("twofaRemember").(bool) - u, err := user_model.GetUserByID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - if ctx.Session.Get("linkAccount") != nil { - if err := externalaccount.LinkAccountFromStore(ctx.Session, u); err != nil { - ctx.ServerError("UserSignIn", err) - } - } - - twofa.LastUsedPasscode = form.Passcode - if err = login.UpdateTwoFactor(twofa); err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - handleSignIn(ctx, u, remember) - return - } - - ctx.RenderWithErr(ctx.Tr("auth.twofa_passcode_incorrect"), tplTwofa, forms.TwoFactorAuthForm{}) -} - -// TwoFactorScratch shows the scratch code form for two-factor authentication. -func TwoFactorScratch(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("twofa_scratch") - - // Check auto-login. - if checkAutoLogin(ctx) { - return - } - - // Ensure user is in a 2FA session. - if ctx.Session.Get("twofaUid") == nil { - ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) - return - } - - ctx.HTML(http.StatusOK, tplTwofaScratch) -} - -// TwoFactorScratchPost validates and invalidates a user's two-factor scratch token. -func TwoFactorScratchPost(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.TwoFactorScratchAuthForm) - ctx.Data["Title"] = ctx.Tr("twofa_scratch") - - // Ensure user is in a 2FA session. - idSess := ctx.Session.Get("twofaUid") - if idSess == nil { - ctx.ServerError("UserSignIn", errors.New("not in 2FA session")) - return - } - - id := idSess.(int64) - twofa, err := login.GetTwoFactorByUID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - // Validate the passcode with the stored TOTP secret. - if twofa.VerifyScratchToken(form.Token) { - // Invalidate the scratch token. - _, err = twofa.GenerateScratchToken() - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - if err = login.UpdateTwoFactor(twofa); err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - remember := ctx.Session.Get("twofaRemember").(bool) - u, err := user_model.GetUserByID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - handleSignInFull(ctx, u, remember, false) - if ctx.Written() { - return - } - ctx.Flash.Info(ctx.Tr("auth.twofa_scratch_used")) - ctx.Redirect(setting.AppSubURL + "/user/settings/security") - return - } - - ctx.RenderWithErr(ctx.Tr("auth.twofa_scratch_token_incorrect"), tplTwofaScratch, forms.TwoFactorScratchAuthForm{}) -} - -// U2F shows the U2F login page -func U2F(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("twofa") - ctx.Data["RequireU2F"] = true - // Check auto-login. - if checkAutoLogin(ctx) { - return - } - - // Ensure user is in a 2FA session. - if ctx.Session.Get("twofaUid") == nil { - ctx.ServerError("UserSignIn", errors.New("not in U2F session")) - return - } - - // See whether TOTP is also available. - if ctx.Session.Get("totpEnrolled") != nil { - ctx.Data["TOTPEnrolled"] = true - } - - ctx.HTML(http.StatusOK, tplU2F) -} - -// U2FChallenge submits a sign challenge to the browser -func U2FChallenge(ctx *context.Context) { - // Ensure user is in a U2F session. - idSess := ctx.Session.Get("twofaUid") - if idSess == nil { - ctx.ServerError("UserSignIn", errors.New("not in U2F session")) - return - } - id := idSess.(int64) - regs, err := login.GetU2FRegistrationsByUID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - if len(regs) == 0 { - ctx.ServerError("UserSignIn", errors.New("no device registered")) - return - } - challenge, err := u2f.NewChallenge(setting.U2F.AppID, setting.U2F.TrustedFacets) - if err != nil { - ctx.ServerError("u2f.NewChallenge", err) - return - } - if err := ctx.Session.Set("u2fChallenge", challenge); err != nil { - ctx.ServerError("UserSignIn: unable to set u2fChallenge in session", err) - return - } - if err := ctx.Session.Release(); err != nil { - ctx.ServerError("UserSignIn: unable to store session", err) - } - - ctx.JSON(http.StatusOK, challenge.SignRequest(regs.ToRegistrations())) -} - -// U2FSign authenticates the user by signResp -func U2FSign(ctx *context.Context) { - signResp := web.GetForm(ctx).(*u2f.SignResponse) - challSess := ctx.Session.Get("u2fChallenge") - idSess := ctx.Session.Get("twofaUid") - if challSess == nil || idSess == nil { - ctx.ServerError("UserSignIn", errors.New("not in U2F session")) - return - } - challenge := challSess.(*u2f.Challenge) - id := idSess.(int64) - regs, err := login.GetU2FRegistrationsByUID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - for _, reg := range regs { - r, err := reg.Parse() - if err != nil { - log.Error("parsing u2f registration: %v", err) - continue - } - newCounter, authErr := r.Authenticate(*signResp, *challenge, reg.Counter) - if authErr == nil { - reg.Counter = newCounter - user, err := user_model.GetUserByID(id) - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - remember := ctx.Session.Get("twofaRemember").(bool) - if err := reg.UpdateCounter(); err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - if ctx.Session.Get("linkAccount") != nil { - if err := externalaccount.LinkAccountFromStore(ctx.Session, user); err != nil { - ctx.ServerError("UserSignIn", err) - } - } - redirect := handleSignInFull(ctx, user, remember, false) - if ctx.Written() { - return - } - if redirect == "" { - redirect = setting.AppSubURL + "/" - } - ctx.PlainText(http.StatusOK, redirect) - return - } - } - ctx.Error(http.StatusUnauthorized) -} - -// This handles the final part of the sign-in process of the user. -func handleSignIn(ctx *context.Context, u *user_model.User, remember bool) { - redirect := handleSignInFull(ctx, u, remember, true) - if ctx.Written() { - return - } - ctx.Redirect(redirect) -} - -func handleSignInFull(ctx *context.Context, u *user_model.User, remember, obeyRedirect bool) string { - if remember { - days := 86400 * setting.LogInRememberDays - ctx.SetCookie(setting.CookieUserName, u.Name, days) - ctx.SetSuperSecureCookie(base.EncodeMD5(u.Rands+u.Passwd), - setting.CookieRememberName, u.Name, days) - } - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("RegenerateSession", err) - return setting.AppSubURL + "/" - } - - // Delete the openid, 2fa and linkaccount data - _ = ctx.Session.Delete("openid_verified_uri") - _ = ctx.Session.Delete("openid_signin_remember") - _ = ctx.Session.Delete("openid_determined_email") - _ = ctx.Session.Delete("openid_determined_username") - _ = ctx.Session.Delete("twofaUid") - _ = ctx.Session.Delete("twofaRemember") - _ = ctx.Session.Delete("u2fChallenge") - _ = ctx.Session.Delete("linkAccount") - if err := ctx.Session.Set("uid", u.ID); err != nil { - log.Error("Error setting uid %d in session: %v", u.ID, err) - } - if err := ctx.Session.Set("uname", u.Name); err != nil { - log.Error("Error setting uname %s session: %v", u.Name, err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Unable to store session: %v", err) - } - - // Language setting of the user overwrites the one previously set - // If the user does not have a locale set, we save the current one. - if len(u.Language) == 0 { - u.Language = ctx.Locale.Language() - if err := user_model.UpdateUserCols(db.DefaultContext, u, "language"); err != nil { - ctx.ServerError("UpdateUserCols Language", fmt.Errorf("Error updating user language [user: %d, locale: %s]", u.ID, u.Language)) - return setting.AppSubURL + "/" - } - } - - middleware.SetLocaleCookie(ctx.Resp, u.Language, 0) - - if ctx.Locale.Language() != u.Language { - ctx.Locale = middleware.Locale(ctx.Resp, ctx.Req) - } - - // Clear whatever CSRF has right now, force to generate a new one - middleware.DeleteCSRFCookie(ctx.Resp) - - // Register last login - u.SetLastLogin() - if err := user_model.UpdateUserCols(db.DefaultContext, u, "last_login_unix"); err != nil { - ctx.ServerError("UpdateUserCols", err) - return setting.AppSubURL + "/" - } - - if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 && !utils.IsExternalURL(redirectTo) { - middleware.DeleteRedirectToCookie(ctx.Resp) - if obeyRedirect { - ctx.RedirectToFirst(redirectTo) - } - return redirectTo - } - - if obeyRedirect { - ctx.Redirect(setting.AppSubURL + "/") - } - return setting.AppSubURL + "/" -} - -// SignInOAuth handles the OAuth2 login buttons -func SignInOAuth(ctx *context.Context) { - provider := ctx.Params(":provider") - - loginSource, err := login.GetActiveOAuth2LoginSourceByName(provider) - if err != nil { - ctx.ServerError("SignIn", err) - return - } - - // try to do a direct callback flow, so we don't authenticate the user again but use the valid accesstoken to get the user - user, gothUser, err := oAuth2UserLoginCallback(loginSource, ctx.Req, ctx.Resp) - if err == nil && user != nil { - // we got the user without going through the whole OAuth2 authentication flow again - handleOAuth2SignIn(ctx, loginSource, user, gothUser) - return - } - - if err = loginSource.Cfg.(*oauth2.Source).Callout(ctx.Req, ctx.Resp); err != nil { - if strings.Contains(err.Error(), "no provider for ") { - if err = oauth2.ResetOAuth2(); err != nil { - ctx.ServerError("SignIn", err) - return - } - if err = loginSource.Cfg.(*oauth2.Source).Callout(ctx.Req, ctx.Resp); err != nil { - ctx.ServerError("SignIn", err) - } - return - } - ctx.ServerError("SignIn", err) - } - // redirect is done in oauth2.Auth -} - -// SignInOAuthCallback handles the callback from the given provider -func SignInOAuthCallback(ctx *context.Context) { - provider := ctx.Params(":provider") - - // first look if the provider is still active - loginSource, err := login.GetActiveOAuth2LoginSourceByName(provider) - if err != nil { - ctx.ServerError("SignIn", err) - return - } - - if loginSource == nil { - ctx.ServerError("SignIn", errors.New("No valid provider found, check configured callback url in provider")) - return - } - - u, gothUser, err := oAuth2UserLoginCallback(loginSource, ctx.Req, ctx.Resp) - - if err != nil { - if user_model.IsErrUserProhibitLogin(err) { - uplerr := err.(*user_model.ErrUserProhibitLogin) - log.Info("Failed authentication attempt for %s from %s: %v", uplerr.Name, ctx.RemoteAddr(), err) - ctx.Data["Title"] = ctx.Tr("auth.prohibit_login") - ctx.HTML(http.StatusOK, "user/auth/prohibit_login") - return - } - ctx.ServerError("UserSignIn", err) - return - } - - if u == nil { - if !setting.Service.AllowOnlyInternalRegistration && setting.OAuth2Client.EnableAutoRegistration { - // create new user with details from oauth2 provider - var missingFields []string - if gothUser.UserID == "" { - missingFields = append(missingFields, "sub") - } - if gothUser.Email == "" { - missingFields = append(missingFields, "email") - } - if setting.OAuth2Client.Username == setting.OAuth2UsernameNickname && gothUser.NickName == "" { - missingFields = append(missingFields, "nickname") - } - if len(missingFields) > 0 { - log.Error("OAuth2 Provider %s returned empty or missing fields: %s", loginSource.Name, missingFields) - if loginSource.IsOAuth2() && loginSource.Cfg.(*oauth2.Source).Provider == "openidConnect" { - log.Error("You may need to change the 'OPENID_CONNECT_SCOPES' setting to request all required fields") - } - err = fmt.Errorf("OAuth2 Provider %s returned empty or missing fields: %s", loginSource.Name, missingFields) - ctx.ServerError("CreateUser", err) - return - } - u = &user_model.User{ - Name: getUserName(&gothUser), - FullName: gothUser.Name, - Email: gothUser.Email, - IsActive: !setting.OAuth2Client.RegisterEmailConfirm, - LoginType: login.OAuth2, - LoginSource: loginSource.ID, - LoginName: gothUser.UserID, - IsRestricted: setting.Service.DefaultUserIsRestricted, - } - - setUserGroupClaims(loginSource, u, &gothUser) - - if !createAndHandleCreatedUser(ctx, base.TplName(""), nil, u, &gothUser, setting.OAuth2Client.AccountLinking != setting.OAuth2AccountLinkingDisabled) { - // error already handled - return - } - } else { - // no existing user is found, request attach or new account - showLinkingLogin(ctx, gothUser) - return - } - } - - handleOAuth2SignIn(ctx, loginSource, u, gothUser) -} - -func claimValueToStringSlice(claimValue interface{}) []string { - var groups []string - - switch rawGroup := claimValue.(type) { - case []string: - groups = rawGroup - default: - str := fmt.Sprintf("%s", rawGroup) - groups = strings.Split(str, ",") - } - return groups -} - -func setUserGroupClaims(loginSource *login.Source, u *user_model.User, gothUser *goth.User) bool { - - source := loginSource.Cfg.(*oauth2.Source) - if source.GroupClaimName == "" || (source.AdminGroup == "" && source.RestrictedGroup == "") { - return false - } - - groupClaims, has := gothUser.RawData[source.GroupClaimName] - if !has { - return false - } - - groups := claimValueToStringSlice(groupClaims) - - wasAdmin, wasRestricted := u.IsAdmin, u.IsRestricted - - if source.AdminGroup != "" { - u.IsAdmin = false - } - if source.RestrictedGroup != "" { - u.IsRestricted = false - } - - for _, g := range groups { - if source.AdminGroup != "" && g == source.AdminGroup { - u.IsAdmin = true - } else if source.RestrictedGroup != "" && g == source.RestrictedGroup { - u.IsRestricted = true - } - } - - return wasAdmin != u.IsAdmin || wasRestricted != u.IsRestricted -} - -func getUserName(gothUser *goth.User) string { - switch setting.OAuth2Client.Username { - case setting.OAuth2UsernameEmail: - return strings.Split(gothUser.Email, "@")[0] - case setting.OAuth2UsernameNickname: - return gothUser.NickName - default: // OAuth2UsernameUserid - return gothUser.UserID - } -} - -func showLinkingLogin(ctx *context.Context, gothUser goth.User) { - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("RegenerateSession", err) - return - } - - if err := ctx.Session.Set("linkAccountGothUser", gothUser); err != nil { - log.Error("Error setting linkAccountGothUser in session: %v", err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Error storing session: %v", err) - } - ctx.Redirect(setting.AppSubURL + "/user/link_account") -} - -func updateAvatarIfNeed(url string, u *user_model.User) { - if setting.OAuth2Client.UpdateAvatar && len(url) > 0 { - resp, err := http.Get(url) - if err == nil { - defer func() { - _ = resp.Body.Close() - }() - } - // ignore any error - if err == nil && resp.StatusCode == http.StatusOK { - data, err := io.ReadAll(io.LimitReader(resp.Body, setting.Avatar.MaxFileSize+1)) - if err == nil && int64(len(data)) <= setting.Avatar.MaxFileSize { - _ = user_service.UploadAvatar(u, data) - } - } - } -} - -func handleOAuth2SignIn(ctx *context.Context, source *login.Source, u *user_model.User, gothUser goth.User) { - updateAvatarIfNeed(gothUser.AvatarURL, u) - - needs2FA := false - if !source.Cfg.(*oauth2.Source).SkipLocalTwoFA { - _, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { - ctx.ServerError("UserSignIn", err) - return - } - needs2FA = err == nil - } - - // If this user is enrolled in 2FA and this source doesn't override it, - // we can't sign the user in just yet. Instead, redirect them to the 2FA authentication page. - if !needs2FA { - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("RegenerateSession", err) - return - } - - // Set session IDs - if err := ctx.Session.Set("uid", u.ID); err != nil { - log.Error("Error setting uid in session: %v", err) - } - if err := ctx.Session.Set("uname", u.Name); err != nil { - log.Error("Error setting uname in session: %v", err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Error storing session: %v", err) - } - - // Clear whatever CSRF has right now, force to generate a new one - middleware.DeleteCSRFCookie(ctx.Resp) - - // Register last login - u.SetLastLogin() - - // Update GroupClaims - changed := setUserGroupClaims(source, u, &gothUser) - cols := []string{"last_login_unix"} - if changed { - cols = append(cols, "is_admin", "is_restricted") - } - - if err := user_model.UpdateUserCols(db.DefaultContext, u, cols...); err != nil { - ctx.ServerError("UpdateUserCols", err) - return - } - - // update external user information - if err := externalaccount.UpdateExternalUser(u, gothUser); err != nil { - log.Error("UpdateExternalUser failed: %v", err) - } - - if err := resetLocale(ctx, u); err != nil { - ctx.ServerError("resetLocale", err) - return - } - - if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 { - middleware.DeleteRedirectToCookie(ctx.Resp) - ctx.RedirectToFirst(redirectTo) - return - } - - ctx.Redirect(setting.AppSubURL + "/") - return - } - - changed := setUserGroupClaims(source, u, &gothUser) - if changed { - if err := user_model.UpdateUserCols(db.DefaultContext, u, "is_admin", "is_restricted"); err != nil { - ctx.ServerError("UpdateUserCols", err) - return - } - } - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("RegenerateSession", err) - return - } - - // User needs to use 2FA, save data and redirect to 2FA page. - if err := ctx.Session.Set("twofaUid", u.ID); err != nil { - log.Error("Error setting twofaUid in session: %v", err) - } - if err := ctx.Session.Set("twofaRemember", false); err != nil { - log.Error("Error setting twofaRemember in session: %v", err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Error storing session: %v", err) - } - - // If U2F is enrolled -> Redirect to U2F instead - regs, err := login.GetU2FRegistrationsByUID(u.ID) - if err == nil && len(regs) > 0 { - ctx.Redirect(setting.AppSubURL + "/user/u2f") - return - } - - ctx.Redirect(setting.AppSubURL + "/user/two_factor") -} - -// OAuth2UserLoginCallback attempts to handle the callback from the OAuth2 provider and if successful -// login the user -func oAuth2UserLoginCallback(loginSource *login.Source, request *http.Request, response http.ResponseWriter) (*user_model.User, goth.User, error) { - oauth2Source := loginSource.Cfg.(*oauth2.Source) - - gothUser, err := oauth2Source.Callback(request, response) - if err != nil { - if err.Error() == "securecookie: the value is too long" || strings.Contains(err.Error(), "Data too long") { - log.Error("OAuth2 Provider %s returned too long a token. Current max: %d. Either increase the [OAuth2] MAX_TOKEN_LENGTH or reduce the information returned from the OAuth2 provider", loginSource.Name, setting.OAuth2.MaxTokenLength) - err = fmt.Errorf("OAuth2 Provider %s returned too long a token. Current max: %d. Either increase the [OAuth2] MAX_TOKEN_LENGTH or reduce the information returned from the OAuth2 provider", loginSource.Name, setting.OAuth2.MaxTokenLength) - } - return nil, goth.User{}, err - } - - if oauth2Source.RequiredClaimName != "" { - claimInterface, has := gothUser.RawData[oauth2Source.RequiredClaimName] - if !has { - return nil, goth.User{}, user_model.ErrUserProhibitLogin{Name: gothUser.UserID} - } - - if oauth2Source.RequiredClaimValue != "" { - groups := claimValueToStringSlice(claimInterface) - found := false - for _, group := range groups { - if group == oauth2Source.RequiredClaimValue { - found = true - break - } - } - if !found { - return nil, goth.User{}, user_model.ErrUserProhibitLogin{Name: gothUser.UserID} - } - } - } - - user := &user_model.User{ - LoginName: gothUser.UserID, - LoginType: login.OAuth2, - LoginSource: loginSource.ID, - } - - hasUser, err := user_model.GetUser(user) - if err != nil { - return nil, goth.User{}, err - } - - if hasUser { - return user, gothUser, nil - } - - // search in external linked users - externalLoginUser := &user_model.ExternalLoginUser{ - ExternalID: gothUser.UserID, - LoginSourceID: loginSource.ID, - } - hasUser, err = user_model.GetExternalLogin(externalLoginUser) - if err != nil { - return nil, goth.User{}, err - } - if hasUser { - user, err = user_model.GetUserByID(externalLoginUser.UserID) - return user, gothUser, err - } - - // no user found to login - return nil, gothUser, nil - -} - -// LinkAccount shows the page where the user can decide to login or create a new account -func LinkAccount(ctx *context.Context) { - ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration - ctx.Data["Title"] = ctx.Tr("link_account") - ctx.Data["LinkAccountMode"] = true - ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha - ctx.Data["Captcha"] = context.GetImageCaptcha() - ctx.Data["CaptchaType"] = setting.Service.CaptchaType - ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL - ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey - ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey - ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration - ctx.Data["AllowOnlyInternalRegistration"] = setting.Service.AllowOnlyInternalRegistration - ctx.Data["ShowRegistrationButton"] = false - - // use this to set the right link into the signIn and signUp templates in the link_account template - ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" - ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" - - gothUser := ctx.Session.Get("linkAccountGothUser") - if gothUser == nil { - ctx.ServerError("UserSignIn", errors.New("not in LinkAccount session")) - return - } - - gu, _ := gothUser.(goth.User) - uname := getUserName(&gu) - email := gu.Email - ctx.Data["user_name"] = uname - ctx.Data["email"] = email - - if len(email) != 0 { - u, err := user_model.GetUserByEmail(email) - if err != nil && !user_model.IsErrUserNotExist(err) { - ctx.ServerError("UserSignIn", err) - return - } - if u != nil { - ctx.Data["user_exists"] = true - } - } else if len(uname) != 0 { - u, err := user_model.GetUserByName(uname) - if err != nil && !user_model.IsErrUserNotExist(err) { - ctx.ServerError("UserSignIn", err) - return - } - if u != nil { - ctx.Data["user_exists"] = true - } - } - - ctx.HTML(http.StatusOK, tplLinkAccount) -} - -// LinkAccountPostSignIn handle the coupling of external account with another account using signIn -func LinkAccountPostSignIn(ctx *context.Context) { - signInForm := web.GetForm(ctx).(*forms.SignInForm) - ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration - ctx.Data["Title"] = ctx.Tr("link_account") - ctx.Data["LinkAccountMode"] = true - ctx.Data["LinkAccountModeSignIn"] = true - ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha - ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL - ctx.Data["Captcha"] = context.GetImageCaptcha() - ctx.Data["CaptchaType"] = setting.Service.CaptchaType - ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey - ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey - ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration - ctx.Data["ShowRegistrationButton"] = false - - // use this to set the right link into the signIn and signUp templates in the link_account template - ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" - ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" - - gothUser := ctx.Session.Get("linkAccountGothUser") - if gothUser == nil { - ctx.ServerError("UserSignIn", errors.New("not in LinkAccount session")) - return - } - - if ctx.HasError() { - ctx.HTML(http.StatusOK, tplLinkAccount) - return - } - - u, _, err := auth.UserSignIn(signInForm.UserName, signInForm.Password) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.Data["user_exists"] = true - ctx.RenderWithErr(ctx.Tr("form.username_password_incorrect"), tplLinkAccount, &signInForm) - } else { - ctx.ServerError("UserLinkAccount", err) - } - return - } - - linkAccount(ctx, u, gothUser.(goth.User), signInForm.Remember) -} - -func linkAccount(ctx *context.Context, u *user_model.User, gothUser goth.User, remember bool) { - updateAvatarIfNeed(gothUser.AvatarURL, u) - - // If this user is enrolled in 2FA, we can't sign the user in just yet. - // Instead, redirect them to the 2FA authentication page. - // We deliberately ignore the skip local 2fa setting here because we are linking to a previous user here - _, err := login.GetTwoFactorByUID(u.ID) - if err != nil { - if !login.IsErrTwoFactorNotEnrolled(err) { - ctx.ServerError("UserLinkAccount", err) - return - } - - err = externalaccount.LinkAccountToUser(u, gothUser) - if err != nil { - ctx.ServerError("UserLinkAccount", err) - return - } - - handleSignIn(ctx, u, remember) - return - } - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - ctx.ServerError("RegenerateSession", err) - return - } - - // User needs to use 2FA, save data and redirect to 2FA page. - if err := ctx.Session.Set("twofaUid", u.ID); err != nil { - log.Error("Error setting twofaUid in session: %v", err) - } - if err := ctx.Session.Set("twofaRemember", remember); err != nil { - log.Error("Error setting twofaRemember in session: %v", err) - } - if err := ctx.Session.Set("linkAccount", true); err != nil { - log.Error("Error setting linkAccount in session: %v", err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Error storing session: %v", err) - } - - // If U2F is enrolled -> Redirect to U2F instead - regs, err := login.GetU2FRegistrationsByUID(u.ID) - if err == nil && len(regs) > 0 { - ctx.Redirect(setting.AppSubURL + "/user/u2f") - return - } - - ctx.Redirect(setting.AppSubURL + "/user/two_factor") -} - -// LinkAccountPostRegister handle the creation of a new account for an external account using signUp -func LinkAccountPostRegister(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.RegisterForm) - // TODO Make insecure passwords optional for local accounts also, - // once email-based Second-Factor Auth is available - ctx.Data["DisablePassword"] = !setting.Service.RequireExternalRegistrationPassword || setting.Service.AllowOnlyExternalRegistration - ctx.Data["Title"] = ctx.Tr("link_account") - ctx.Data["LinkAccountMode"] = true - ctx.Data["LinkAccountModeRegister"] = true - ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha - ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL - ctx.Data["Captcha"] = context.GetImageCaptcha() - ctx.Data["CaptchaType"] = setting.Service.CaptchaType - ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey - ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey - ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration - ctx.Data["ShowRegistrationButton"] = false - - // use this to set the right link into the signIn and signUp templates in the link_account template - ctx.Data["SignInLink"] = setting.AppSubURL + "/user/link_account_signin" - ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/link_account_signup" - - gothUserInterface := ctx.Session.Get("linkAccountGothUser") - if gothUserInterface == nil { - ctx.ServerError("UserSignUp", errors.New("not in LinkAccount session")) - return - } - gothUser, ok := gothUserInterface.(goth.User) - if !ok { - ctx.ServerError("UserSignUp", fmt.Errorf("session linkAccountGothUser type is %t but not goth.User", gothUserInterface)) - return - } - - if ctx.HasError() { - ctx.HTML(http.StatusOK, tplLinkAccount) - return - } - - if setting.Service.DisableRegistration || setting.Service.AllowOnlyInternalRegistration { - ctx.Error(http.StatusForbidden) - return - } - - if setting.Service.EnableCaptcha && setting.Service.RequireExternalRegistrationCaptcha { - var valid bool - var err error - switch setting.Service.CaptchaType { - case setting.ImageCaptcha: - valid = context.GetImageCaptcha().VerifyReq(ctx.Req) - case setting.ReCaptcha: - valid, err = recaptcha.Verify(ctx, form.GRecaptchaResponse) - case setting.HCaptcha: - valid, err = hcaptcha.Verify(ctx, form.HcaptchaResponse) - default: - ctx.ServerError("Unknown Captcha Type", fmt.Errorf("Unknown Captcha Type: %s", setting.Service.CaptchaType)) - return - } - if err != nil { - log.Debug("%s", err.Error()) - } - - if !valid { - ctx.Data["Err_Captcha"] = true - ctx.RenderWithErr(ctx.Tr("form.captcha_incorrect"), tplLinkAccount, &form) - return - } - } - - if !form.IsEmailDomainAllowed() { - ctx.RenderWithErr(ctx.Tr("auth.email_domain_blacklisted"), tplLinkAccount, &form) - return - } - - if setting.Service.AllowOnlyExternalRegistration || !setting.Service.RequireExternalRegistrationPassword { - // In user_model.User an empty password is classed as not set, so we set form.Password to empty. - // Eventually the database should be changed to indicate "Second Factor"-enabled accounts - // (accounts that do not introduce the security vulnerabilities of a password). - // If a user decides to circumvent second-factor security, and purposefully create a password, - // they can still do so using the "Recover Account" option. - form.Password = "" - } else { - if (len(strings.TrimSpace(form.Password)) > 0 || len(strings.TrimSpace(form.Retype)) > 0) && form.Password != form.Retype { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplLinkAccount, &form) - return - } - if len(strings.TrimSpace(form.Password)) > 0 && len(form.Password) < setting.MinPasswordLength { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplLinkAccount, &form) - return - } - } - - loginSource, err := login.GetActiveOAuth2LoginSourceByName(gothUser.Provider) - if err != nil { - ctx.ServerError("CreateUser", err) - } - - u := &user_model.User{ - Name: form.UserName, - Email: form.Email, - Passwd: form.Password, - IsActive: !(setting.Service.RegisterEmailConfirm || setting.Service.RegisterManualConfirm), - LoginType: login.OAuth2, - LoginSource: loginSource.ID, - LoginName: gothUser.UserID, - } - - if !createAndHandleCreatedUser(ctx, tplLinkAccount, form, u, &gothUser, false) { - // error already handled - return - } - - handleSignIn(ctx, u, false) -} - -// HandleSignOut resets the session and sets the cookies -func HandleSignOut(ctx *context.Context) { - _ = ctx.Session.Flush() - _ = ctx.Session.Destroy(ctx.Resp, ctx.Req) - ctx.DeleteCookie(setting.CookieUserName) - ctx.DeleteCookie(setting.CookieRememberName) - middleware.DeleteCSRFCookie(ctx.Resp) - middleware.DeleteLocaleCookie(ctx.Resp) - middleware.DeleteRedirectToCookie(ctx.Resp) -} - -// SignOut sign out from login status -func SignOut(ctx *context.Context) { - if ctx.User != nil { - eventsource.GetManager().SendMessageBlocking(ctx.User.ID, &eventsource.Event{ - Name: "logout", - Data: ctx.Session.ID(), - }) - } - HandleSignOut(ctx) - ctx.Redirect(setting.AppSubURL + "/") -} - -// SignUp render the register page -func SignUp(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("sign_up") - - ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/sign_up" - - ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha - ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL - ctx.Data["Captcha"] = context.GetImageCaptcha() - ctx.Data["CaptchaType"] = setting.Service.CaptchaType - ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey - ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey - ctx.Data["PageIsSignUp"] = true - - //Show Disabled Registration message if DisableRegistration or AllowOnlyExternalRegistration options are true - ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration - - ctx.HTML(http.StatusOK, tplSignUp) -} - -// SignUpPost response for sign up information submission -func SignUpPost(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.RegisterForm) - ctx.Data["Title"] = ctx.Tr("sign_up") - - ctx.Data["SignUpLink"] = setting.AppSubURL + "/user/sign_up" - - ctx.Data["EnableCaptcha"] = setting.Service.EnableCaptcha - ctx.Data["RecaptchaURL"] = setting.Service.RecaptchaURL - ctx.Data["Captcha"] = context.GetImageCaptcha() - ctx.Data["CaptchaType"] = setting.Service.CaptchaType - ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey - ctx.Data["HcaptchaSitekey"] = setting.Service.HcaptchaSitekey - ctx.Data["PageIsSignUp"] = true - - //Permission denied if DisableRegistration or AllowOnlyExternalRegistration options are true - if setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration { - ctx.Error(http.StatusForbidden) - return - } - - if ctx.HasError() { - ctx.HTML(http.StatusOK, tplSignUp) - return - } - - if setting.Service.EnableCaptcha { - var valid bool - var err error - switch setting.Service.CaptchaType { - case setting.ImageCaptcha: - valid = context.GetImageCaptcha().VerifyReq(ctx.Req) - case setting.ReCaptcha: - valid, err = recaptcha.Verify(ctx, form.GRecaptchaResponse) - case setting.HCaptcha: - valid, err = hcaptcha.Verify(ctx, form.HcaptchaResponse) - default: - ctx.ServerError("Unknown Captcha Type", fmt.Errorf("Unknown Captcha Type: %s", setting.Service.CaptchaType)) - return - } - if err != nil { - log.Debug("%s", err.Error()) - } - - if !valid { - ctx.Data["Err_Captcha"] = true - ctx.RenderWithErr(ctx.Tr("form.captcha_incorrect"), tplSignUp, &form) - return - } - } - - if !form.IsEmailDomainAllowed() { - ctx.RenderWithErr(ctx.Tr("auth.email_domain_blacklisted"), tplSignUp, &form) - return - } - - if form.Password != form.Retype { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplSignUp, &form) - return - } - if len(form.Password) < setting.MinPasswordLength { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplSignUp, &form) - return - } - if !password.IsComplexEnough(form.Password) { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(password.BuildComplexityError(ctx), tplSignUp, &form) - return - } - pwned, err := password.IsPwned(ctx, form.Password) - if pwned { - errMsg := ctx.Tr("auth.password_pwned") - if err != nil { - log.Error(err.Error()) - errMsg = ctx.Tr("auth.password_pwned_err") - } - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(errMsg, tplSignUp, &form) - return - } - - u := &user_model.User{ - Name: form.UserName, - Email: form.Email, - Passwd: form.Password, - IsActive: !(setting.Service.RegisterEmailConfirm || setting.Service.RegisterManualConfirm), - IsRestricted: setting.Service.DefaultUserIsRestricted, - } - - if !createAndHandleCreatedUser(ctx, tplSignUp, form, u, nil, false) { - // error already handled - return - } - - ctx.Flash.Success(ctx.Tr("auth.sign_up_successful")) - handleSignIn(ctx, u, false) -} - -// createAndHandleCreatedUser calls createUserInContext and -// then handleUserCreated. -func createAndHandleCreatedUser(ctx *context.Context, tpl base.TplName, form interface{}, u *user_model.User, gothUser *goth.User, allowLink bool) bool { - if !createUserInContext(ctx, tpl, form, u, gothUser, allowLink) { - return false - } - return handleUserCreated(ctx, u, gothUser) -} - -// createUserInContext creates a user and handles errors within a given context. -// Optionally a template can be specified. -func createUserInContext(ctx *context.Context, tpl base.TplName, form interface{}, u *user_model.User, gothUser *goth.User, allowLink bool) (ok bool) { - if err := user_model.CreateUser(u); err != nil { - if allowLink && (user_model.IsErrUserAlreadyExist(err) || user_model.IsErrEmailAlreadyUsed(err)) { - if setting.OAuth2Client.AccountLinking == setting.OAuth2AccountLinkingAuto { - var user *user_model.User - user = &user_model.User{Name: u.Name} - hasUser, err := user_model.GetUser(user) - if !hasUser || err != nil { - user = &user_model.User{Email: u.Email} - hasUser, err = user_model.GetUser(user) - if !hasUser || err != nil { - ctx.ServerError("UserLinkAccount", err) - return - } - } - - // TODO: probably we should respect 'remember' user's choice... - linkAccount(ctx, user, *gothUser, true) - return // user is already created here, all redirects are handled - } else if setting.OAuth2Client.AccountLinking == setting.OAuth2AccountLinkingLogin { - showLinkingLogin(ctx, *gothUser) - return // user will be created only after linking login - } - } - - // handle error without template - if len(tpl) == 0 { - ctx.ServerError("CreateUser", err) - return - } - - // handle error with template - switch { - case user_model.IsErrUserAlreadyExist(err): - ctx.Data["Err_UserName"] = true - ctx.RenderWithErr(ctx.Tr("form.username_been_taken"), tpl, form) - case user_model.IsErrEmailAlreadyUsed(err): - ctx.Data["Err_Email"] = true - ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tpl, form) - case user_model.IsErrEmailInvalid(err): - ctx.Data["Err_Email"] = true - ctx.RenderWithErr(ctx.Tr("form.email_invalid"), tpl, form) - case db.IsErrNameReserved(err): - ctx.Data["Err_UserName"] = true - ctx.RenderWithErr(ctx.Tr("user.form.name_reserved", err.(db.ErrNameReserved).Name), tpl, form) - case db.IsErrNamePatternNotAllowed(err): - ctx.Data["Err_UserName"] = true - ctx.RenderWithErr(ctx.Tr("user.form.name_pattern_not_allowed", err.(db.ErrNamePatternNotAllowed).Pattern), tpl, form) - case db.IsErrNameCharsNotAllowed(err): - ctx.Data["Err_UserName"] = true - ctx.RenderWithErr(ctx.Tr("user.form.name_chars_not_allowed", err.(db.ErrNameCharsNotAllowed).Name), tpl, form) - default: - ctx.ServerError("CreateUser", err) - } - return - } - log.Trace("Account created: %s", u.Name) - return true -} - -// handleUserCreated does additional steps after a new user is created. -// It auto-sets admin for the only user, updates the optional external user and -// sends a confirmation email if required. -func handleUserCreated(ctx *context.Context, u *user_model.User, gothUser *goth.User) (ok bool) { - // Auto-set admin for the only user. - if user_model.CountUsers() == 1 { - u.IsAdmin = true - u.IsActive = true - u.SetLastLogin() - if err := user_model.UpdateUserCols(db.DefaultContext, u, "is_admin", "is_active", "last_login_unix"); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - } - - // update external user information - if gothUser != nil { - if err := externalaccount.UpdateExternalUser(u, *gothUser); err != nil { - log.Error("UpdateExternalUser failed: %v", err) - } - } - - // Send confirmation email - if !u.IsActive && u.ID > 1 { - mailer.SendActivateAccountMail(ctx.Locale, u) - - ctx.Data["IsSendRegisterMail"] = true - ctx.Data["Email"] = u.Email - ctx.Data["ActiveCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language()) - ctx.HTML(http.StatusOK, TplActivate) - - if err := ctx.Cache.Put("MailResendLimit_"+u.LowerName, u.LowerName, 180); err != nil { - log.Error("Set cache(MailResendLimit) fail: %v", err) - } - return - } - - return true -} - -// Activate render activate user page -func Activate(ctx *context.Context) { - code := ctx.FormString("code") - - if len(code) == 0 { - ctx.Data["IsActivatePage"] = true - if ctx.User == nil || ctx.User.IsActive { - ctx.NotFound("invalid user", nil) - return - } - // Resend confirmation email. - if setting.Service.RegisterEmailConfirm { - if ctx.Cache.IsExist("MailResendLimit_" + ctx.User.LowerName) { - ctx.Data["ResendLimited"] = true - } else { - ctx.Data["ActiveCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language()) - mailer.SendActivateAccountMail(ctx.Locale, ctx.User) - - if err := ctx.Cache.Put("MailResendLimit_"+ctx.User.LowerName, ctx.User.LowerName, 180); err != nil { - log.Error("Set cache(MailResendLimit) fail: %v", err) - } - } - } else { - ctx.Data["ServiceNotEnabled"] = true - } - ctx.HTML(http.StatusOK, TplActivate) - return - } - - user := user_model.VerifyUserActiveCode(code) - // if code is wrong - if user == nil { - ctx.Data["IsActivateFailed"] = true - ctx.HTML(http.StatusOK, TplActivate) - return - } - - // if account is local account, verify password - if user.LoginSource == 0 { - ctx.Data["Code"] = code - ctx.Data["NeedsPassword"] = true - ctx.HTML(http.StatusOK, TplActivate) - return - } - - handleAccountActivation(ctx, user) -} - -// ActivatePost handles account activation with password check -func ActivatePost(ctx *context.Context) { - code := ctx.FormString("code") - if len(code) == 0 { - ctx.Redirect(setting.AppSubURL + "/user/activate") - return - } - - user := user_model.VerifyUserActiveCode(code) - // if code is wrong - if user == nil { - ctx.Data["IsActivateFailed"] = true - ctx.HTML(http.StatusOK, TplActivate) - return - } - - // if account is local account, verify password - if user.LoginSource == 0 { - password := ctx.FormString("password") - if len(password) == 0 { - ctx.Data["Code"] = code - ctx.Data["NeedsPassword"] = true - ctx.HTML(http.StatusOK, TplActivate) - return - } - if !user.ValidatePassword(password) { - ctx.Data["IsActivateFailed"] = true - ctx.HTML(http.StatusOK, TplActivate) - return - } - } - - handleAccountActivation(ctx, user) -} - -func handleAccountActivation(ctx *context.Context, user *user_model.User) { - user.IsActive = true - var err error - if user.Rands, err = user_model.GetUserSalt(); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - if err := user_model.UpdateUserCols(db.DefaultContext, user, "is_active", "rands"); err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.NotFound("UpdateUserCols", err) - } else { - ctx.ServerError("UpdateUser", err) - } - return - } - - if err := user_model.ActivateUserEmail(user.ID, user.Email, true); err != nil { - log.Error("Unable to activate email for user: %-v with email: %s: %v", user, user.Email, err) - ctx.ServerError("ActivateUserEmail", err) - return - } - - log.Trace("User activated: %s", user.Name) - - if _, err := session.RegenerateSession(ctx.Resp, ctx.Req); err != nil { - log.Error("Unable to regenerate session for user: %-v with email: %s: %v", user, user.Email, err) - ctx.ServerError("ActivateUserEmail", err) - return - } - - // Set session IDs - if err := ctx.Session.Set("uid", user.ID); err != nil { - log.Error("Error setting uid in session[%s]: %v", ctx.Session.ID(), err) - } - if err := ctx.Session.Set("uname", user.Name); err != nil { - log.Error("Error setting uname in session[%s]: %v", ctx.Session.ID(), err) - } - if err := ctx.Session.Release(); err != nil { - log.Error("Error storing session[%s]: %v", ctx.Session.ID(), err) - } - - if err := resetLocale(ctx, user); err != nil { - ctx.ServerError("resetLocale", err) - return - } - - ctx.Flash.Success(ctx.Tr("auth.account_activated")) - ctx.Redirect(setting.AppSubURL + "/") -} - -// ActivateEmail render the activate email page -func ActivateEmail(ctx *context.Context) { - code := ctx.FormString("code") - emailStr := ctx.FormString("email") - - // Verify code. - if email := user_model.VerifyActiveEmailCode(code, emailStr); email != nil { - if err := user_model.ActivateEmail(email); err != nil { - ctx.ServerError("ActivateEmail", err) - } - - log.Trace("Email activated: %s", email.Email) - ctx.Flash.Success(ctx.Tr("settings.add_email_success")) - - if u, err := user_model.GetUserByID(email.UID); err != nil { - log.Warn("GetUserByID: %d", email.UID) - } else { - // Allow user to validate more emails - _ = ctx.Cache.Delete("MailResendLimit_" + u.LowerName) - } - } - - // FIXME: e-mail verification does not require the user to be logged in, - // so this could be redirecting to the login page. - // Should users be logged in automatically here? (consider 2FA requirements, etc.) - ctx.Redirect(setting.AppSubURL + "/user/settings/account") -} - -// ForgotPasswd render the forget password page -func ForgotPasswd(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("auth.forgot_password_title") - - if setting.MailService == nil { - log.Warn(ctx.Tr("auth.disable_forgot_password_mail_admin")) - ctx.Data["IsResetDisable"] = true - ctx.HTML(http.StatusOK, tplForgotPassword) - return - } - - ctx.Data["Email"] = ctx.FormString("email") - - ctx.Data["IsResetRequest"] = true - ctx.HTML(http.StatusOK, tplForgotPassword) -} - -// ForgotPasswdPost response for forget password request -func ForgotPasswdPost(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("auth.forgot_password_title") - - if setting.MailService == nil { - ctx.NotFound("ForgotPasswdPost", nil) - return - } - ctx.Data["IsResetRequest"] = true - - email := ctx.FormString("email") - ctx.Data["Email"] = email - - u, err := user_model.GetUserByEmail(email) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.Data["ResetPwdCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, ctx.Locale.Language()) - ctx.Data["IsResetSent"] = true - ctx.HTML(http.StatusOK, tplForgotPassword) - return - } - - ctx.ServerError("user.ResetPasswd(check existence)", err) - return - } - - if !u.IsLocal() && !u.IsOAuth2() { - ctx.Data["Err_Email"] = true - ctx.RenderWithErr(ctx.Tr("auth.non_local_account"), tplForgotPassword, nil) - return - } - - if ctx.Cache.IsExist("MailResendLimit_" + u.LowerName) { - ctx.Data["ResendLimited"] = true - ctx.HTML(http.StatusOK, tplForgotPassword) - return - } - - mailer.SendResetPasswordMail(u) - - if err = ctx.Cache.Put("MailResendLimit_"+u.LowerName, u.LowerName, 180); err != nil { - log.Error("Set cache(MailResendLimit) fail: %v", err) - } - - ctx.Data["ResetPwdCodeLives"] = timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, ctx.Locale.Language()) - ctx.Data["IsResetSent"] = true - ctx.HTML(http.StatusOK, tplForgotPassword) -} - -func commonResetPassword(ctx *context.Context) (*user_model.User, *login.TwoFactor) { - code := ctx.FormString("code") - - ctx.Data["Title"] = ctx.Tr("auth.reset_password") - ctx.Data["Code"] = code - - if nil != ctx.User { - ctx.Data["user_signed_in"] = true - } - - if len(code) == 0 { - ctx.Flash.Error(ctx.Tr("auth.invalid_code")) - return nil, nil - } - - // Fail early, don't frustrate the user - u := user_model.VerifyUserActiveCode(code) - if u == nil { - ctx.Flash.Error(ctx.Tr("auth.invalid_code")) - return nil, nil - } - - twofa, err := login.GetTwoFactorByUID(u.ID) - if err != nil { - if !login.IsErrTwoFactorNotEnrolled(err) { - ctx.Error(http.StatusInternalServerError, "CommonResetPassword", err.Error()) - return nil, nil - } - } else { - ctx.Data["has_two_factor"] = true - ctx.Data["scratch_code"] = ctx.FormBool("scratch_code") - } - - // Show the user that they are affecting the account that they intended to - ctx.Data["user_email"] = u.Email - - if nil != ctx.User && u.ID != ctx.User.ID { - ctx.Flash.Error(ctx.Tr("auth.reset_password_wrong_user", ctx.User.Email, u.Email)) - return nil, nil - } - - return u, twofa -} - -// ResetPasswd render the account recovery page -func ResetPasswd(ctx *context.Context) { - ctx.Data["IsResetForm"] = true - - commonResetPassword(ctx) - if ctx.Written() { - return - } - - ctx.HTML(http.StatusOK, tplResetPassword) -} - -// ResetPasswdPost response from account recovery request -func ResetPasswdPost(ctx *context.Context) { - u, twofa := commonResetPassword(ctx) - if ctx.Written() { - return - } - - if u == nil { - // Flash error has been set - ctx.HTML(http.StatusOK, tplResetPassword) - return - } - - // Validate password length. - passwd := ctx.FormString("password") - if len(passwd) < setting.MinPasswordLength { - ctx.Data["IsResetForm"] = true - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplResetPassword, nil) - return - } else if !password.IsComplexEnough(passwd) { - ctx.Data["IsResetForm"] = true - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(password.BuildComplexityError(ctx), tplResetPassword, nil) - return - } else if pwned, err := password.IsPwned(ctx, passwd); pwned || err != nil { - errMsg := ctx.Tr("auth.password_pwned") - if err != nil { - log.Error(err.Error()) - errMsg = ctx.Tr("auth.password_pwned_err") - } - ctx.Data["IsResetForm"] = true - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(errMsg, tplResetPassword, nil) - return - } - - // Handle two-factor - regenerateScratchToken := false - if twofa != nil { - if ctx.FormBool("scratch_code") { - if !twofa.VerifyScratchToken(ctx.FormString("token")) { - ctx.Data["IsResetForm"] = true - ctx.Data["Err_Token"] = true - ctx.RenderWithErr(ctx.Tr("auth.twofa_scratch_token_incorrect"), tplResetPassword, nil) - return - } - regenerateScratchToken = true - } else { - passcode := ctx.FormString("passcode") - ok, err := twofa.ValidateTOTP(passcode) - if err != nil { - ctx.Error(http.StatusInternalServerError, "ValidateTOTP", err.Error()) - return - } - if !ok || twofa.LastUsedPasscode == passcode { - ctx.Data["IsResetForm"] = true - ctx.Data["Err_Passcode"] = true - ctx.RenderWithErr(ctx.Tr("auth.twofa_passcode_incorrect"), tplResetPassword, nil) - return - } - - twofa.LastUsedPasscode = passcode - if err = login.UpdateTwoFactor(twofa); err != nil { - ctx.ServerError("ResetPasswdPost: UpdateTwoFactor", err) - return - } - } - } - var err error - if u.Rands, err = user_model.GetUserSalt(); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - if err = u.SetPassword(passwd); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - u.MustChangePassword = false - if err := user_model.UpdateUserCols(db.DefaultContext, u, "must_change_password", "passwd", "passwd_hash_algo", "rands", "salt"); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - - log.Trace("User password reset: %s", u.Name) - ctx.Data["IsResetFailed"] = true - remember := len(ctx.FormString("remember")) != 0 - - if regenerateScratchToken { - // Invalidate the scratch token. - _, err = twofa.GenerateScratchToken() - if err != nil { - ctx.ServerError("UserSignIn", err) - return - } - if err = login.UpdateTwoFactor(twofa); err != nil { - ctx.ServerError("UserSignIn", err) - return - } - - handleSignInFull(ctx, u, remember, false) - ctx.Flash.Info(ctx.Tr("auth.twofa_scratch_used")) - if ctx.Written() { - return - } - ctx.Redirect(setting.AppSubURL + "/user/settings/security") - return - } - - handleSignIn(ctx, u, remember) -} - -// MustChangePassword renders the page to change a user's password -func MustChangePassword(ctx *context.Context) { - ctx.Data["Title"] = ctx.Tr("auth.must_change_password") - ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/settings/change_password" - ctx.Data["MustChangePassword"] = true - ctx.HTML(http.StatusOK, tplMustChangePassword) -} - -// MustChangePasswordPost response for updating a user's password after his/her -// account was created by an admin -func MustChangePasswordPost(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.MustChangePasswordForm) - ctx.Data["Title"] = ctx.Tr("auth.must_change_password") - ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/settings/change_password" - if ctx.HasError() { - ctx.HTML(http.StatusOK, tplMustChangePassword) - return - } - u := ctx.User - // Make sure only requests for users who are eligible to change their password via - // this method passes through - if !u.MustChangePassword { - ctx.ServerError("MustUpdatePassword", errors.New("cannot update password.. Please visit the settings page")) - return - } - - if form.Password != form.Retype { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("form.password_not_match"), tplMustChangePassword, &form) - return - } - - if len(form.Password) < setting.MinPasswordLength { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(ctx.Tr("auth.password_too_short", setting.MinPasswordLength), tplMustChangePassword, &form) - return - } - if !password.IsComplexEnough(form.Password) { - ctx.Data["Err_Password"] = true - ctx.RenderWithErr(password.BuildComplexityError(ctx), tplMustChangePassword, &form) - return - } - pwned, err := password.IsPwned(ctx, form.Password) - if pwned { - ctx.Data["Err_Password"] = true - errMsg := ctx.Tr("auth.password_pwned") - if err != nil { - log.Error(err.Error()) - errMsg = ctx.Tr("auth.password_pwned_err") - } - ctx.RenderWithErr(errMsg, tplMustChangePassword, &form) - return - } - - if err = u.SetPassword(form.Password); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - - u.MustChangePassword = false - - if err := user_model.UpdateUserCols(db.DefaultContext, u, "must_change_password", "passwd", "passwd_hash_algo", "salt"); err != nil { - ctx.ServerError("UpdateUser", err) - return - } - - ctx.Flash.Success(ctx.Tr("settings.change_password_success")) - - log.Trace("User updated password: %s", u.Name) - - if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 && !utils.IsExternalURL(redirectTo) { - middleware.DeleteRedirectToCookie(ctx.Resp) - ctx.RedirectToFirst(redirectTo) - return - } - - ctx.Redirect(setting.AppSubURL + "/") -} diff --git a/routers/web/user/setting/applications.go b/routers/web/user/setting/applications.go index 9976337bfab9..20ffdfaf840e 100644 --- a/routers/web/user/setting/applications.go +++ b/routers/web/user/setting/applications.go @@ -9,7 +9,7 @@ import ( "net/http" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" @@ -93,12 +93,12 @@ func loadApplicationsData(ctx *context.Context) { ctx.Data["Tokens"] = tokens ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable if setting.OAuth2.Enable { - ctx.Data["Applications"], err = login.GetOAuth2ApplicationsByUserID(ctx.User.ID) + ctx.Data["Applications"], err = auth.GetOAuth2ApplicationsByUserID(ctx.User.ID) if err != nil { ctx.ServerError("GetOAuth2ApplicationsByUserID", err) return } - ctx.Data["Grants"], err = login.GetOAuth2GrantsByUserID(ctx.User.ID) + ctx.Data["Grants"], err = auth.GetOAuth2GrantsByUserID(ctx.User.ID) if err != nil { ctx.ServerError("GetOAuth2GrantsByUserID", err) return diff --git a/routers/web/user/setting/keys.go b/routers/web/user/setting/keys.go index 8274f8b1b3b2..26bbf7c19533 100644 --- a/routers/web/user/setting/keys.go +++ b/routers/web/user/setting/keys.go @@ -216,7 +216,6 @@ func KeysPost(ctx *context.Context) { // DeleteKey response for delete user's SSH/GPG key func DeleteKey(ctx *context.Context) { - switch ctx.FormString("type") { case "gpg": if err := asymkey_model.DeleteGPGKey(ctx.User, ctx.FormInt64("id")); err != nil { diff --git a/routers/web/user/setting/oauth2.go b/routers/web/user/setting/oauth2.go index 0f338ab5d1cb..65d0924da0e6 100644 --- a/routers/web/user/setting/oauth2.go +++ b/routers/web/user/setting/oauth2.go @@ -8,7 +8,7 @@ import ( "fmt" "net/http" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -34,7 +34,7 @@ func OAuthApplicationsPost(ctx *context.Context) { return } // TODO validate redirect URI - app, err := login.CreateOAuth2Application(login.CreateOAuth2ApplicationOptions{ + app, err := auth.CreateOAuth2Application(auth.CreateOAuth2ApplicationOptions{ Name: form.Name, RedirectURIs: []string{form.RedirectURI}, UserID: ctx.User.ID, @@ -67,7 +67,7 @@ func OAuthApplicationsEdit(ctx *context.Context) { } // TODO validate redirect URI var err error - if ctx.Data["App"], err = login.UpdateOAuth2Application(login.UpdateOAuth2ApplicationOptions{ + if ctx.Data["App"], err = auth.UpdateOAuth2Application(auth.UpdateOAuth2ApplicationOptions{ ID: ctx.ParamsInt64("id"), Name: form.Name, RedirectURIs: []string{form.RedirectURI}, @@ -85,9 +85,9 @@ func OAuthApplicationsRegenerateSecret(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("settings") ctx.Data["PageIsSettingsApplications"] = true - app, err := login.GetOAuth2ApplicationByID(ctx.ParamsInt64("id")) + app, err := auth.GetOAuth2ApplicationByID(ctx.ParamsInt64("id")) if err != nil { - if login.IsErrOAuthApplicationNotFound(err) { + if auth.IsErrOAuthApplicationNotFound(err) { ctx.NotFound("Application not found", err) return } @@ -110,9 +110,9 @@ func OAuthApplicationsRegenerateSecret(ctx *context.Context) { // OAuth2ApplicationShow displays the given application func OAuth2ApplicationShow(ctx *context.Context) { - app, err := login.GetOAuth2ApplicationByID(ctx.ParamsInt64("id")) + app, err := auth.GetOAuth2ApplicationByID(ctx.ParamsInt64("id")) if err != nil { - if login.IsErrOAuthApplicationNotFound(err) { + if auth.IsErrOAuthApplicationNotFound(err) { ctx.NotFound("Application not found", err) return } @@ -129,7 +129,7 @@ func OAuth2ApplicationShow(ctx *context.Context) { // DeleteOAuth2Application deletes the given oauth2 application func DeleteOAuth2Application(ctx *context.Context) { - if err := login.DeleteOAuth2Application(ctx.FormInt64("id"), ctx.User.ID); err != nil { + if err := auth.DeleteOAuth2Application(ctx.FormInt64("id"), ctx.User.ID); err != nil { ctx.ServerError("DeleteOAuth2Application", err) return } @@ -147,7 +147,7 @@ func RevokeOAuth2Grant(ctx *context.Context) { ctx.ServerError("RevokeOAuth2Grant", fmt.Errorf("user id or grant id is zero")) return } - if err := login.RevokeOAuth2Grant(ctx.FormInt64("id"), ctx.User.ID); err != nil { + if err := auth.RevokeOAuth2Grant(ctx.FormInt64("id"), ctx.User.ID); err != nil { ctx.ServerError("RevokeOAuth2Grant", err) return } diff --git a/routers/web/user/setting/security_twofa.go b/routers/web/user/setting/security/2fa.go similarity index 91% rename from routers/web/user/setting/security_twofa.go rename to routers/web/user/setting/security/2fa.go index 94f975f9fe6f..98e1f9b083ea 100644 --- a/routers/web/user/setting/security_twofa.go +++ b/routers/web/user/setting/security/2fa.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package setting +package security import ( "bytes" @@ -13,7 +13,7 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -29,9 +29,9 @@ func RegenerateScratchTwoFactor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("settings") ctx.Data["PageIsSettingsSecurity"] = true - t, err := login.GetTwoFactorByUID(ctx.User.ID) + t, err := auth.GetTwoFactorByUID(ctx.User.ID) if err != nil { - if login.IsErrTwoFactorNotEnrolled(err) { + if auth.IsErrTwoFactorNotEnrolled(err) { ctx.Flash.Error(ctx.Tr("settings.twofa_not_enrolled")) ctx.Redirect(setting.AppSubURL + "/user/settings/security") } @@ -45,7 +45,7 @@ func RegenerateScratchTwoFactor(ctx *context.Context) { return } - if err = login.UpdateTwoFactor(t); err != nil { + if err = auth.UpdateTwoFactor(t); err != nil { ctx.ServerError("SettingsTwoFactor: Failed to UpdateTwoFactor", err) return } @@ -59,9 +59,9 @@ func DisableTwoFactor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("settings") ctx.Data["PageIsSettingsSecurity"] = true - t, err := login.GetTwoFactorByUID(ctx.User.ID) + t, err := auth.GetTwoFactorByUID(ctx.User.ID) if err != nil { - if login.IsErrTwoFactorNotEnrolled(err) { + if auth.IsErrTwoFactorNotEnrolled(err) { ctx.Flash.Error(ctx.Tr("settings.twofa_not_enrolled")) ctx.Redirect(setting.AppSubURL + "/user/settings/security") } @@ -69,8 +69,8 @@ func DisableTwoFactor(ctx *context.Context) { return } - if err = login.DeleteTwoFactorByID(t.ID, ctx.User.ID); err != nil { - if login.IsErrTwoFactorNotEnrolled(err) { + if err = auth.DeleteTwoFactorByID(t.ID, ctx.User.ID); err != nil { + if auth.IsErrTwoFactorNotEnrolled(err) { // There is a potential DB race here - we must have been disabled by another request in the intervening period ctx.Flash.Success(ctx.Tr("settings.twofa_disabled")) ctx.Redirect(setting.AppSubURL + "/user/settings/security") @@ -146,7 +146,7 @@ func EnrollTwoFactor(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("settings") ctx.Data["PageIsSettingsSecurity"] = true - t, err := login.GetTwoFactorByUID(ctx.User.ID) + t, err := auth.GetTwoFactorByUID(ctx.User.ID) if t != nil { // already enrolled - we should redirect back! log.Warn("Trying to re-enroll %-v in twofa when already enrolled", ctx.User) @@ -154,7 +154,7 @@ func EnrollTwoFactor(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/user/settings/security") return } - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { ctx.ServerError("SettingsTwoFactor: GetTwoFactorByUID", err) return } @@ -172,14 +172,14 @@ func EnrollTwoFactorPost(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("settings") ctx.Data["PageIsSettingsSecurity"] = true - t, err := login.GetTwoFactorByUID(ctx.User.ID) + t, err := auth.GetTwoFactorByUID(ctx.User.ID) if t != nil { // already enrolled ctx.Flash.Error(ctx.Tr("settings.twofa_is_enrolled")) ctx.Redirect(setting.AppSubURL + "/user/settings/security") return } - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { ctx.ServerError("SettingsTwoFactor: Failed to check if already enrolled with GetTwoFactorByUID", err) return } @@ -209,7 +209,7 @@ func EnrollTwoFactorPost(ctx *context.Context) { return } - t = &login.TwoFactor{ + t = &auth.TwoFactor{ UID: ctx.User.ID, } err = t.SetSecret(secret) @@ -238,7 +238,7 @@ func EnrollTwoFactorPost(ctx *context.Context) { log.Error("Unable to save changes to the session: %v", err) } - if err = login.NewTwoFactor(t); err != nil { + if err = auth.NewTwoFactor(t); err != nil { // FIXME: We need to handle a unique constraint fail here it's entirely possible that another request has beaten us. // If there is a unique constraint fail we should just tolerate the error ctx.ServerError("SettingsTwoFactor: Failed to save two factor", err) diff --git a/routers/web/user/setting/security_openid.go b/routers/web/user/setting/security/openid.go similarity index 99% rename from routers/web/user/setting/security_openid.go rename to routers/web/user/setting/security/openid.go index 9cdda79b92ed..1a15a2414c38 100644 --- a/routers/web/user/setting/security_openid.go +++ b/routers/web/user/setting/security/openid.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package setting +package security import ( "net/http" diff --git a/routers/web/user/setting/security.go b/routers/web/user/setting/security/security.go similarity index 76% rename from routers/web/user/setting/security.go rename to routers/web/user/setting/security/security.go index d34de519b7e6..0ca6503f05a4 100644 --- a/routers/web/user/setting/security.go +++ b/routers/web/user/setting/security/security.go @@ -3,13 +3,13 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package setting +package security import ( "net/http" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" @@ -17,8 +17,8 @@ import ( ) const ( - tplSettingsSecurity base.TplName = "user/settings/security" - tplSettingsTwofaEnroll base.TplName = "user/settings/twofa_enroll" + tplSettingsSecurity base.TplName = "user/settings/security/security" + tplSettingsTwofaEnroll base.TplName = "user/settings/security/twofa_enroll" ) // Security render change user's password page and 2FA @@ -56,14 +56,14 @@ func DeleteAccountLink(ctx *context.Context) { } func loadSecurityData(ctx *context.Context) { - enrolled, err := login.HasTwoFactorByUID(ctx.User.ID) + enrolled, err := auth.HasTwoFactorByUID(ctx.User.ID) if err != nil { ctx.ServerError("SettingsTwoFactor", err) return } ctx.Data["TOTPEnrolled"] = enrolled - ctx.Data["U2FRegistrations"], err = login.GetU2FRegistrationsByUID(ctx.User.ID) + ctx.Data["U2FRegistrations"], err = auth.GetU2FRegistrationsByUID(ctx.User.ID) if err != nil { ctx.ServerError("GetU2FRegistrationsByUID", err) return @@ -82,10 +82,10 @@ func loadSecurityData(ctx *context.Context) { return } - // map the provider display name with the LoginSource - sources := make(map[*login.Source]string) + // map the provider display name with the AuthSource + sources := make(map[*auth.Source]string) for _, externalAccount := range accountLinks { - if loginSource, err := login.GetSourceByID(externalAccount.LoginSourceID); err == nil { + if authSource, err := auth.GetSourceByID(externalAccount.LoginSourceID); err == nil { var providerDisplayName string type DisplayNamed interface { @@ -96,14 +96,14 @@ func loadSecurityData(ctx *context.Context) { Name() string } - if displayNamed, ok := loginSource.Cfg.(DisplayNamed); ok { + if displayNamed, ok := authSource.Cfg.(DisplayNamed); ok { providerDisplayName = displayNamed.DisplayName() - } else if named, ok := loginSource.Cfg.(Named); ok { + } else if named, ok := authSource.Cfg.(Named); ok { providerDisplayName = named.Name() } else { - providerDisplayName = loginSource.Name + providerDisplayName = authSource.Name } - sources[loginSource] = providerDisplayName + sources[authSource] = providerDisplayName } } ctx.Data["AccountLinks"] = sources diff --git a/routers/web/user/setting/security_u2f.go b/routers/web/user/setting/security/u2f.go similarity index 89% rename from routers/web/user/setting/security_u2f.go rename to routers/web/user/setting/security/u2f.go index d1d6d1e8cad8..dd35a893f447 100644 --- a/routers/web/user/setting/security_u2f.go +++ b/routers/web/user/setting/security/u2f.go @@ -2,13 +2,13 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package setting +package security import ( "errors" "net/http" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -34,7 +34,7 @@ func U2FRegister(ctx *context.Context) { ctx.ServerError("Unable to set session key for u2fChallenge", err) return } - regs, err := login.GetU2FRegistrationsByUID(ctx.User.ID) + regs, err := auth.GetU2FRegistrationsByUID(ctx.User.ID) if err != nil { ctx.ServerError("GetU2FRegistrationsByUID", err) return @@ -78,7 +78,7 @@ func U2FRegisterPost(ctx *context.Context) { ctx.ServerError("u2f.Register", err) return } - if _, err = login.CreateRegistration(ctx.User.ID, name, reg); err != nil { + if _, err = auth.CreateRegistration(ctx.User.ID, name, reg); err != nil { ctx.ServerError("u2f.Register", err) return } @@ -88,9 +88,9 @@ func U2FRegisterPost(ctx *context.Context) { // U2FDelete deletes an security key by id func U2FDelete(ctx *context.Context) { form := web.GetForm(ctx).(*forms.U2FDeleteForm) - reg, err := login.GetU2FRegistrationByID(form.ID) + reg, err := auth.GetU2FRegistrationByID(form.ID) if err != nil { - if login.IsErrU2FRegistrationNotExist(err) { + if auth.IsErrU2FRegistrationNotExist(err) { ctx.Status(200) return } @@ -101,7 +101,7 @@ func U2FDelete(ctx *context.Context) { ctx.Status(401) return } - if err := login.DeleteRegistration(reg); err != nil { + if err := auth.DeleteRegistration(reg); err != nil { ctx.ServerError("DeleteRegistration", err) return } diff --git a/routers/web/web.go b/routers/web/web.go index 486255096dc0..3589ec9bd663 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -24,14 +24,16 @@ import ( "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/api/v1/misc" "code.gitea.io/gitea/routers/web/admin" + "code.gitea.io/gitea/routers/web/auth" "code.gitea.io/gitea/routers/web/dev" "code.gitea.io/gitea/routers/web/events" "code.gitea.io/gitea/routers/web/explore" "code.gitea.io/gitea/routers/web/org" "code.gitea.io/gitea/routers/web/repo" "code.gitea.io/gitea/routers/web/user" - userSetting "code.gitea.io/gitea/routers/web/user/setting" - "code.gitea.io/gitea/services/auth" + user_setting "code.gitea.io/gitea/routers/web/user/setting" + "code.gitea.io/gitea/routers/web/user/setting/security" + auth_service "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/forms" "code.gitea.io/gitea/services/lfs" "code.gitea.io/gitea/services/mailer" @@ -154,7 +156,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { common = append(common, context.Contexter()) // Get user from session if logged in. - common = append(common, context.Auth(auth.NewGroup(auth.Methods()...))) + common = append(common, context.Auth(auth_service.NewGroup(auth_service.Methods()...))) // GetHead allows a HEAD request redirect to GET if HEAD method is not defined for that route common = append(common, middleware.GetHead) @@ -233,7 +235,7 @@ func RegisterRoutes(m *web.Route) { // for health check m.Get("/", Home) m.Group("/.well-known", func() { - m.Get("/openid-configuration", user.OIDCWellKnown) + m.Get("/openid-configuration", auth.OIDCWellKnown) if setting.Federation.Enabled { m.Get("/nodeinfo", NodeInfoLinks) } @@ -257,42 +259,42 @@ func RegisterRoutes(m *web.Route) { // ***** START: User ***** m.Group("/user", func() { - m.Get("/login", user.SignIn) - m.Post("/login", bindIgnErr(forms.SignInForm{}), user.SignInPost) + m.Get("/login", auth.SignIn) + m.Post("/login", bindIgnErr(forms.SignInForm{}), auth.SignInPost) m.Group("", func() { m.Combo("/login/openid"). - Get(user.SignInOpenID). - Post(bindIgnErr(forms.SignInOpenIDForm{}), user.SignInOpenIDPost) + Get(auth.SignInOpenID). + Post(bindIgnErr(forms.SignInOpenIDForm{}), auth.SignInOpenIDPost) }, openIDSignInEnabled) m.Group("/openid", func() { m.Combo("/connect"). - Get(user.ConnectOpenID). - Post(bindIgnErr(forms.ConnectOpenIDForm{}), user.ConnectOpenIDPost) + Get(auth.ConnectOpenID). + Post(bindIgnErr(forms.ConnectOpenIDForm{}), auth.ConnectOpenIDPost) m.Group("/register", func() { m.Combo(""). - Get(user.RegisterOpenID, openIDSignUpEnabled). - Post(bindIgnErr(forms.SignUpOpenIDForm{}), user.RegisterOpenIDPost) + Get(auth.RegisterOpenID, openIDSignUpEnabled). + Post(bindIgnErr(forms.SignUpOpenIDForm{}), auth.RegisterOpenIDPost) }, openIDSignUpEnabled) }, openIDSignInEnabled) - m.Get("/sign_up", user.SignUp) - m.Post("/sign_up", bindIgnErr(forms.RegisterForm{}), user.SignUpPost) + m.Get("/sign_up", auth.SignUp) + m.Post("/sign_up", bindIgnErr(forms.RegisterForm{}), auth.SignUpPost) m.Group("/oauth2", func() { - m.Get("/{provider}", user.SignInOAuth) - m.Get("/{provider}/callback", user.SignInOAuthCallback) + m.Get("/{provider}", auth.SignInOAuth) + m.Get("/{provider}/callback", auth.SignInOAuthCallback) }) - m.Get("/link_account", user.LinkAccount) - m.Post("/link_account_signin", bindIgnErr(forms.SignInForm{}), user.LinkAccountPostSignIn) - m.Post("/link_account_signup", bindIgnErr(forms.RegisterForm{}), user.LinkAccountPostRegister) + m.Get("/link_account", auth.LinkAccount) + m.Post("/link_account_signin", bindIgnErr(forms.SignInForm{}), auth.LinkAccountPostSignIn) + m.Post("/link_account_signup", bindIgnErr(forms.RegisterForm{}), auth.LinkAccountPostRegister) m.Group("/two_factor", func() { - m.Get("", user.TwoFactor) - m.Post("", bindIgnErr(forms.TwoFactorAuthForm{}), user.TwoFactorPost) - m.Get("/scratch", user.TwoFactorScratch) - m.Post("/scratch", bindIgnErr(forms.TwoFactorScratchAuthForm{}), user.TwoFactorScratchPost) + m.Get("", auth.TwoFactor) + m.Post("", bindIgnErr(forms.TwoFactorAuthForm{}), auth.TwoFactorPost) + m.Get("/scratch", auth.TwoFactorScratch) + m.Post("/scratch", bindIgnErr(forms.TwoFactorScratchAuthForm{}), auth.TwoFactorScratchPost) }) m.Group("/u2f", func() { - m.Get("", user.U2F) - m.Get("/challenge", user.U2FChallenge) - m.Post("/sign", bindIgnErr(u2f.SignResponse{}), user.U2FSign) + m.Get("", auth.U2F) + m.Get("/challenge", auth.U2FChallenge) + m.Post("/sign", bindIgnErr(u2f.SignResponse{}), auth.U2FSign) }) }, reqSignOut) @@ -300,71 +302,71 @@ func RegisterRoutes(m *web.Route) { m.Any("/user/events", events.Events) m.Group("/login/oauth", func() { - m.Get("/authorize", bindIgnErr(forms.AuthorizationForm{}), user.AuthorizeOAuth) - m.Post("/grant", bindIgnErr(forms.GrantApplicationForm{}), user.GrantApplicationOAuth) + m.Get("/authorize", bindIgnErr(forms.AuthorizationForm{}), auth.AuthorizeOAuth) + m.Post("/grant", bindIgnErr(forms.GrantApplicationForm{}), auth.GrantApplicationOAuth) // TODO manage redirection - m.Post("/authorize", bindIgnErr(forms.AuthorizationForm{}), user.AuthorizeOAuth) + m.Post("/authorize", bindIgnErr(forms.AuthorizationForm{}), auth.AuthorizeOAuth) }, ignSignInAndCsrf, reqSignIn) - m.Get("/login/oauth/userinfo", ignSignInAndCsrf, user.InfoOAuth) - m.Post("/login/oauth/access_token", CorsHandler(), bindIgnErr(forms.AccessTokenForm{}), ignSignInAndCsrf, user.AccessTokenOAuth) - m.Get("/login/oauth/keys", ignSignInAndCsrf, user.OIDCKeys) - m.Post("/login/oauth/introspect", CorsHandler(), bindIgnErr(forms.IntrospectTokenForm{}), ignSignInAndCsrf, user.IntrospectOAuth) + m.Get("/login/oauth/userinfo", ignSignInAndCsrf, auth.InfoOAuth) + m.Post("/login/oauth/access_token", CorsHandler(), bindIgnErr(forms.AccessTokenForm{}), ignSignInAndCsrf, auth.AccessTokenOAuth) + m.Get("/login/oauth/keys", ignSignInAndCsrf, auth.OIDCKeys) + m.Post("/login/oauth/introspect", CorsHandler(), bindIgnErr(forms.IntrospectTokenForm{}), ignSignInAndCsrf, auth.IntrospectOAuth) m.Group("/user/settings", func() { - m.Get("", userSetting.Profile) - m.Post("", bindIgnErr(forms.UpdateProfileForm{}), userSetting.ProfilePost) - m.Get("/change_password", user.MustChangePassword) - m.Post("/change_password", bindIgnErr(forms.MustChangePasswordForm{}), user.MustChangePasswordPost) - m.Post("/avatar", bindIgnErr(forms.AvatarForm{}), userSetting.AvatarPost) - m.Post("/avatar/delete", userSetting.DeleteAvatar) + m.Get("", user_setting.Profile) + m.Post("", bindIgnErr(forms.UpdateProfileForm{}), user_setting.ProfilePost) + m.Get("/change_password", auth.MustChangePassword) + m.Post("/change_password", bindIgnErr(forms.MustChangePasswordForm{}), auth.MustChangePasswordPost) + m.Post("/avatar", bindIgnErr(forms.AvatarForm{}), user_setting.AvatarPost) + m.Post("/avatar/delete", user_setting.DeleteAvatar) m.Group("/account", func() { - m.Combo("").Get(userSetting.Account).Post(bindIgnErr(forms.ChangePasswordForm{}), userSetting.AccountPost) - m.Post("/email", bindIgnErr(forms.AddEmailForm{}), userSetting.EmailPost) - m.Post("/email/delete", userSetting.DeleteEmail) - m.Post("/delete", userSetting.DeleteAccount) + m.Combo("").Get(user_setting.Account).Post(bindIgnErr(forms.ChangePasswordForm{}), user_setting.AccountPost) + m.Post("/email", bindIgnErr(forms.AddEmailForm{}), user_setting.EmailPost) + m.Post("/email/delete", user_setting.DeleteEmail) + m.Post("/delete", user_setting.DeleteAccount) }) m.Group("/appearance", func() { - m.Get("", userSetting.Appearance) - m.Post("/language", bindIgnErr(forms.UpdateLanguageForm{}), userSetting.UpdateUserLang) - m.Post("/theme", bindIgnErr(forms.UpdateThemeForm{}), userSetting.UpdateUIThemePost) + m.Get("", user_setting.Appearance) + m.Post("/language", bindIgnErr(forms.UpdateLanguageForm{}), user_setting.UpdateUserLang) + m.Post("/theme", bindIgnErr(forms.UpdateThemeForm{}), user_setting.UpdateUIThemePost) }) m.Group("/security", func() { - m.Get("", userSetting.Security) + m.Get("", security.Security) m.Group("/two_factor", func() { - m.Post("/regenerate_scratch", userSetting.RegenerateScratchTwoFactor) - m.Post("/disable", userSetting.DisableTwoFactor) - m.Get("/enroll", userSetting.EnrollTwoFactor) - m.Post("/enroll", bindIgnErr(forms.TwoFactorAuthForm{}), userSetting.EnrollTwoFactorPost) + m.Post("/regenerate_scratch", security.RegenerateScratchTwoFactor) + m.Post("/disable", security.DisableTwoFactor) + m.Get("/enroll", security.EnrollTwoFactor) + m.Post("/enroll", bindIgnErr(forms.TwoFactorAuthForm{}), security.EnrollTwoFactorPost) }) m.Group("/u2f", func() { - m.Post("/request_register", bindIgnErr(forms.U2FRegistrationForm{}), userSetting.U2FRegister) - m.Post("/register", bindIgnErr(u2f.RegisterResponse{}), userSetting.U2FRegisterPost) - m.Post("/delete", bindIgnErr(forms.U2FDeleteForm{}), userSetting.U2FDelete) + m.Post("/request_register", bindIgnErr(forms.U2FRegistrationForm{}), security.U2FRegister) + m.Post("/register", bindIgnErr(u2f.RegisterResponse{}), security.U2FRegisterPost) + m.Post("/delete", bindIgnErr(forms.U2FDeleteForm{}), security.U2FDelete) }) m.Group("/openid", func() { - m.Post("", bindIgnErr(forms.AddOpenIDForm{}), userSetting.OpenIDPost) - m.Post("/delete", userSetting.DeleteOpenID) - m.Post("/toggle_visibility", userSetting.ToggleOpenIDVisibility) + m.Post("", bindIgnErr(forms.AddOpenIDForm{}), security.OpenIDPost) + m.Post("/delete", security.DeleteOpenID) + m.Post("/toggle_visibility", security.ToggleOpenIDVisibility) }, openIDSignInEnabled) - m.Post("/account_link", userSetting.DeleteAccountLink) + m.Post("/account_link", security.DeleteAccountLink) }) m.Group("/applications/oauth2", func() { - m.Get("/{id}", userSetting.OAuth2ApplicationShow) - m.Post("/{id}", bindIgnErr(forms.EditOAuth2ApplicationForm{}), userSetting.OAuthApplicationsEdit) - m.Post("/{id}/regenerate_secret", userSetting.OAuthApplicationsRegenerateSecret) - m.Post("", bindIgnErr(forms.EditOAuth2ApplicationForm{}), userSetting.OAuthApplicationsPost) - m.Post("/delete", userSetting.DeleteOAuth2Application) - m.Post("/revoke", userSetting.RevokeOAuth2Grant) + m.Get("/{id}", user_setting.OAuth2ApplicationShow) + m.Post("/{id}", bindIgnErr(forms.EditOAuth2ApplicationForm{}), user_setting.OAuthApplicationsEdit) + m.Post("/{id}/regenerate_secret", user_setting.OAuthApplicationsRegenerateSecret) + m.Post("", bindIgnErr(forms.EditOAuth2ApplicationForm{}), user_setting.OAuthApplicationsPost) + m.Post("/delete", user_setting.DeleteOAuth2Application) + m.Post("/revoke", user_setting.RevokeOAuth2Grant) }) - m.Combo("/applications").Get(userSetting.Applications). - Post(bindIgnErr(forms.NewAccessTokenForm{}), userSetting.ApplicationsPost) - m.Post("/applications/delete", userSetting.DeleteApplication) - m.Combo("/keys").Get(userSetting.Keys). - Post(bindIgnErr(forms.AddKeyForm{}), userSetting.KeysPost) - m.Post("/keys/delete", userSetting.DeleteKey) - m.Get("/organization", userSetting.Organization) - m.Get("/repos", userSetting.Repos) - m.Post("/repos/unadopted", userSetting.AdoptOrDeleteRepository) + m.Combo("/applications").Get(user_setting.Applications). + Post(bindIgnErr(forms.NewAccessTokenForm{}), user_setting.ApplicationsPost) + m.Post("/applications/delete", user_setting.DeleteApplication) + m.Combo("/keys").Get(user_setting.Keys). + Post(bindIgnErr(forms.AddKeyForm{}), user_setting.KeysPost) + m.Post("/keys/delete", user_setting.DeleteKey) + m.Get("/organization", user_setting.Organization) + m.Get("/repos", user_setting.Repos) + m.Post("/repos/unadopted", user_setting.AdoptOrDeleteRepository) }, reqSignIn, func(ctx *context.Context) { ctx.Data["PageIsUserSettings"] = true ctx.Data["AllThemes"] = setting.UI.Themes @@ -372,15 +374,15 @@ func RegisterRoutes(m *web.Route) { m.Group("/user", func() { // r.Get("/feeds", binding.Bind(auth.FeedsForm{}), user.Feeds) - m.Get("/activate", user.Activate, reqSignIn) - m.Post("/activate", user.ActivatePost, reqSignIn) - m.Any("/activate_email", user.ActivateEmail) + m.Get("/activate", auth.Activate, reqSignIn) + m.Post("/activate", auth.ActivatePost, reqSignIn) + m.Any("/activate_email", auth.ActivateEmail) m.Get("/avatar/{username}/{size}", user.AvatarByUserName) - m.Get("/recover_account", user.ResetPasswd) - m.Post("/recover_account", user.ResetPasswdPost) - m.Get("/forgot_password", user.ForgotPasswd) - m.Post("/forgot_password", user.ForgotPasswdPost) - m.Post("/logout", user.SignOut) + m.Get("/recover_account", auth.ResetPasswd) + m.Post("/recover_account", auth.ResetPasswdPost) + m.Get("/forgot_password", auth.ForgotPasswd) + m.Post("/forgot_password", auth.ForgotPasswdPost) + m.Post("/logout", auth.SignOut) m.Get("/task/{task}", user.TaskStatus) }) // ***** END: User ***** diff --git a/services/asymkey/sign.go b/services/asymkey/sign.go index 4ccc581803ad..79c5db18a6e8 100644 --- a/services/asymkey/sign.go +++ b/services/asymkey/sign.go @@ -10,8 +10,8 @@ import ( "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -150,8 +150,8 @@ Loop: return false, "", nil, &ErrWontSign{pubkey} } case twofa: - twofaModel, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + twofaModel, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { return false, "", nil, err } if twofaModel == nil { @@ -186,8 +186,8 @@ Loop: return false, "", nil, &ErrWontSign{pubkey} } case twofa: - twofaModel, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + twofaModel, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { return false, "", nil, err } if twofaModel == nil { @@ -239,8 +239,8 @@ Loop: return false, "", nil, &ErrWontSign{pubkey} } case twofa: - twofaModel, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + twofaModel, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { return false, "", nil, err } if twofaModel == nil { @@ -301,8 +301,8 @@ Loop: return false, "", nil, &ErrWontSign{pubkey} } case twofa: - twofaModel, err := login.GetTwoFactorByUID(u.ID) - if err != nil && !login.IsErrTwoFactorNotEnrolled(err) { + twofaModel, err := auth.GetTwoFactorByUID(u.ID) + if err != nil && !auth.IsErrTwoFactorNotEnrolled(err) { return false, "", nil, err } if twofaModel == nil { diff --git a/services/asymkey/ssh_key_test.go b/services/asymkey/ssh_key_test.go index 9de6a4c11bbd..182371271a4c 100644 --- a/services/asymkey/ssh_key_test.go +++ b/services/asymkey/ssh_key_test.go @@ -8,7 +8,7 @@ import ( "testing" asymkey_model "code.gitea.io/gitea/models/asymkey" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" @@ -19,7 +19,7 @@ func TestAddLdapSSHPublicKeys(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}).(*user_model.User) - s := &login.Source{ID: 1} + s := &auth.Source{ID: 1} testCases := []struct { keyString string diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 74dc5eaaf68f..fae981d80078 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -11,8 +11,8 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" @@ -37,8 +37,8 @@ func CheckOAuthAccessToken(accessToken string) int64 { log.Trace("oauth2.ParseToken: %v", err) return 0 } - var grant *login.OAuth2Grant - if grant, err = login.GetOAuth2GrantByID(token.GrantID); err != nil || grant == nil { + var grant *auth.OAuth2Grant + if grant, err = auth.GetOAuth2GrantByID(token.GrantID); err != nil || grant == nil { return 0 } if token.Type != oauth2.TypeAccessToken { diff --git a/services/auth/signin.go b/services/auth/signin.go index 80be419962a5..4392e861f997 100644 --- a/services/auth/signin.go +++ b/services/auth/signin.go @@ -7,8 +7,8 @@ package auth import ( "strings" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/services/auth/source/oauth2" @@ -21,7 +21,7 @@ import ( ) // UserSignIn validates user name and password. -func UserSignIn(username, password string) (*user_model.User, *login.Source, error) { +func UserSignIn(username, password string) (*user_model.User, *auth.Source, error) { var user *user_model.User if strings.Contains(username, "@") { user = &user_model.User{Email: strings.ToLower(strings.TrimSpace(username))} @@ -50,13 +50,13 @@ func UserSignIn(username, password string) (*user_model.User, *login.Source, err } if hasUser { - source, err := login.GetSourceByID(user.LoginSource) + source, err := auth.GetSourceByID(user.LoginSource) if err != nil { return nil, nil, err } if !source.IsActive { - return nil, nil, oauth2.ErrLoginSourceNotActived + return nil, nil, oauth2.ErrAuthSourceNotActived } authenticator, ok := source.Cfg.(PasswordAuthenticator) @@ -78,7 +78,7 @@ func UserSignIn(username, password string) (*user_model.User, *login.Source, err return user, source, nil } - sources, err := login.AllActiveSources() + sources, err := auth.AllActiveSources() if err != nil { return nil, nil, err } diff --git a/services/auth/login_source.go b/services/auth/source.go similarity index 72% rename from services/auth/login_source.go rename to services/auth/source.go index 47a687f63b7f..b7108292d570 100644 --- a/services/auth/login_source.go +++ b/services/auth/source.go @@ -5,18 +5,18 @@ package auth import ( + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" ) -// DeleteLoginSource deletes a LoginSource record in DB. -func DeleteLoginSource(source *login.Source) error { +// DeleteSource deletes a AuthSource record in DB. +func DeleteSource(source *auth.Source) error { count, err := db.GetEngine(db.DefaultContext).Count(&user_model.User{LoginSource: source.ID}) if err != nil { return err } else if count > 0 { - return login.ErrSourceInUse{ + return auth.ErrSourceInUse{ ID: source.ID, } } @@ -25,17 +25,17 @@ func DeleteLoginSource(source *login.Source) error { if err != nil { return err } else if count > 0 { - return login.ErrSourceInUse{ + return auth.ErrSourceInUse{ ID: source.ID, } } - if registerableSource, ok := source.Cfg.(login.RegisterableSource); ok { + if registerableSource, ok := source.Cfg.(auth.RegisterableSource); ok { if err := registerableSource.UnregisterSource(); err != nil { return err } } - _, err = db.GetEngine(db.DefaultContext).ID(source.ID).Delete(new(login.Source)) + _, err = db.GetEngine(db.DefaultContext).ID(source.ID).Delete(new(auth.Source)) return err } diff --git a/services/auth/source/db/assert_interface_test.go b/services/auth/source/db/assert_interface_test.go index a8b137ec4817..f39aaeb1e41b 100644 --- a/services/auth/source/db/assert_interface_test.go +++ b/services/auth/source/db/assert_interface_test.go @@ -5,7 +5,7 @@ package db_test import ( - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/db" ) @@ -15,7 +15,7 @@ import ( type sourceInterface interface { auth.PasswordAuthenticator - login.Config + auth_model.Config } var _ (sourceInterface) = &db.Source{} diff --git a/services/auth/source/db/source.go b/services/auth/source/db/source.go index 5ae2107a3b7d..ecab6d5f3570 100644 --- a/services/auth/source/db/source.go +++ b/services/auth/source/db/source.go @@ -5,7 +5,7 @@ package db import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" ) @@ -29,6 +29,6 @@ func (source *Source) Authenticate(user *user_model.User, login, password string } func init() { - login.RegisterTypeConfig(login.NoType, &Source{}) - login.RegisterTypeConfig(login.Plain, &Source{}) + auth.RegisterTypeConfig(auth.NoType, &Source{}) + auth.RegisterTypeConfig(auth.Plain, &Source{}) } diff --git a/services/auth/source/ldap/assert_interface_test.go b/services/auth/source/ldap/assert_interface_test.go index c480119cd3fe..8fc6903cf37c 100644 --- a/services/auth/source/ldap/assert_interface_test.go +++ b/services/auth/source/ldap/assert_interface_test.go @@ -5,7 +5,7 @@ package ldap_test import ( - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/ldap" ) @@ -17,12 +17,12 @@ type sourceInterface interface { auth.PasswordAuthenticator auth.SynchronizableSource auth.LocalTwoFASkipper - login.SSHKeyProvider - login.Config - login.SkipVerifiable - login.HasTLSer - login.UseTLSer - login.SourceSettable + auth_model.SSHKeyProvider + auth_model.Config + auth_model.SkipVerifiable + auth_model.HasTLSer + auth_model.UseTLSer + auth_model.SourceSettable } var _ (sourceInterface) = &ldap.Source{} diff --git a/services/auth/source/ldap/source.go b/services/auth/source/ldap/source.go index be4a4b2d6226..fc778b0114d7 100644 --- a/services/auth/source/ldap/source.go +++ b/services/auth/source/ldap/source.go @@ -7,7 +7,7 @@ package ldap import ( "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/secret" "code.gitea.io/gitea/modules/setting" @@ -55,8 +55,8 @@ type Source struct { UserUID string // User Attribute listed in Group SkipLocalTwoFA bool `json:",omitempty"` // Skip Local 2fa for users authenticated with this source - // reference to the loginSource - loginSource *login.Source + // reference to the authSource + authSource *auth.Source } // FromDB fills up a LDAPConfig from serialized format. @@ -109,12 +109,12 @@ func (source *Source) ProvidesSSHKeys() bool { return len(strings.TrimSpace(source.AttributeSSHPublicKey)) > 0 } -// SetLoginSource sets the related LoginSource -func (source *Source) SetLoginSource(loginSource *login.Source) { - source.loginSource = loginSource +// SetAuthSource sets the related AuthSource +func (source *Source) SetAuthSource(authSource *auth.Source) { + source.authSource = authSource } func init() { - login.RegisterTypeConfig(login.LDAP, &Source{}) - login.RegisterTypeConfig(login.DLDAP, &Source{}) + auth.RegisterTypeConfig(auth.LDAP, &Source{}) + auth.RegisterTypeConfig(auth.DLDAP, &Source{}) } diff --git a/services/auth/source/ldap/source_authenticate.go b/services/auth/source/ldap/source_authenticate.go index 9938525c0eda..52971bb87e58 100644 --- a/services/auth/source/ldap/source_authenticate.go +++ b/services/auth/source/ldap/source_authenticate.go @@ -9,8 +9,8 @@ import ( "strings" asymkey_model "code.gitea.io/gitea/models/asymkey" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/services/mailer" user_service "code.gitea.io/gitea/services/user" @@ -19,7 +19,7 @@ import ( // Authenticate queries if login/password is valid against the LDAP directory pool, // and create a local user if success when enabled. func (source *Source) Authenticate(user *user_model.User, userName, password string) (*user_model.User, error) { - sr := source.SearchEntry(userName, password, source.loginSource.Type == login.DLDAP) + sr := source.SearchEntry(userName, password, source.authSource.Type == auth.DLDAP) if sr == nil { // User not in LDAP, do nothing return nil, user_model.ErrUserNotExist{Name: userName} @@ -59,7 +59,7 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str } if user != nil { - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(user, source.loginSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(user, source.authSource, sr.SSHPublicKey) { return user, asymkey_model.RewriteAllPublicKeys() } @@ -80,8 +80,8 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str Name: sr.Username, FullName: composeFullName(sr.Name, sr.Surname, sr.Username), Email: sr.Mail, - LoginType: source.loginSource.Type, - LoginSource: source.loginSource.ID, + LoginType: source.authSource.Type, + LoginSource: source.authSource.ID, LoginName: userName, IsActive: true, IsAdmin: sr.IsAdmin, @@ -95,7 +95,7 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str mailer.SendRegisterNotifyMail(user) - if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(user, source.loginSource, sr.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.AddPublicKeysBySource(user, source.authSource, sr.SSHPublicKey) { err = asymkey_model.RewriteAllPublicKeys() } diff --git a/services/auth/source/ldap/source_sync.go b/services/auth/source/ldap/source_sync.go index fb15b2f0467d..78aa90aaa535 100644 --- a/services/auth/source/ldap/source_sync.go +++ b/services/auth/source/ldap/source_sync.go @@ -19,22 +19,22 @@ import ( // Sync causes this ldap source to synchronize its users with the db func (source *Source) Sync(ctx context.Context, updateExisting bool) error { - log.Trace("Doing: SyncExternalUsers[%s]", source.loginSource.Name) + log.Trace("Doing: SyncExternalUsers[%s]", source.authSource.Name) var existingUsers []int isAttributeSSHPublicKeySet := len(strings.TrimSpace(source.AttributeSSHPublicKey)) > 0 var sshKeysNeedUpdate bool // Find all users with this login type - FIXME: Should this be an iterator? - users, err := user_model.GetUsersBySource(source.loginSource) + users, err := user_model.GetUsersBySource(source.authSource) if err != nil { log.Error("SyncExternalUsers: %v", err) return err } select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled before update of %s", source.loginSource.Name) - return db.ErrCancelledf("Before update of %s", source.loginSource.Name) + log.Warn("SyncExternalUsers: Cancelled before update of %s", source.authSource.Name) + return db.ErrCancelledf("Before update of %s", source.authSource.Name) default: } @@ -44,7 +44,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { sr, err := source.SearchEntries() if err != nil { - log.Error("SyncExternalUsers LDAP source failure [%s], skipped", source.loginSource.Name) + log.Error("SyncExternalUsers LDAP source failure [%s], skipped", source.authSource.Name) return nil } @@ -65,7 +65,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { for _, su := range sr { select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled at update of %s before completed update of users", source.loginSource.Name) + log.Warn("SyncExternalUsers: Cancelled at update of %s before completed update of users", source.authSource.Name) // Rewrite authorized_keys file if LDAP Public SSH Key attribute is set and any key was added or removed if sshKeysNeedUpdate { err = asymkey_model.RewriteAllPublicKeys() @@ -73,7 +73,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { log.Error("RewriteAllPublicKeys: %v", err) } } - return db.ErrCancelledf("During update of %s before completed update of users", source.loginSource.Name) + return db.ErrCancelledf("During update of %s before completed update of users", source.authSource.Name) default: } if len(su.Username) == 0 { @@ -96,14 +96,14 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { fullName := composeFullName(su.Name, su.Surname, su.Username) // If no existing user found, create one if usr == nil { - log.Trace("SyncExternalUsers[%s]: Creating user %s", source.loginSource.Name, su.Username) + log.Trace("SyncExternalUsers[%s]: Creating user %s", source.authSource.Name, su.Username) usr = &user_model.User{ LowerName: su.LowerName, Name: su.Username, FullName: fullName, - LoginType: source.loginSource.Type, - LoginSource: source.loginSource.ID, + LoginType: source.authSource.Type, + LoginSource: source.authSource.ID, LoginName: su.Username, Email: su.Mail, IsAdmin: su.IsAdmin, @@ -114,12 +114,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { err = user_model.CreateUser(usr) if err != nil { - log.Error("SyncExternalUsers[%s]: Error creating user %s: %v", source.loginSource.Name, su.Username, err) + log.Error("SyncExternalUsers[%s]: Error creating user %s: %v", source.authSource.Name, su.Username, err) } if err == nil && isAttributeSSHPublicKeySet { - log.Trace("SyncExternalUsers[%s]: Adding LDAP Public SSH Keys for user %s", source.loginSource.Name, usr.Name) - if asymkey_model.AddPublicKeysBySource(usr, source.loginSource, su.SSHPublicKey) { + log.Trace("SyncExternalUsers[%s]: Adding LDAP Public SSH Keys for user %s", source.authSource.Name, usr.Name) + if asymkey_model.AddPublicKeysBySource(usr, source.authSource, su.SSHPublicKey) { sshKeysNeedUpdate = true } } @@ -129,7 +129,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { } } else if updateExisting { // Synchronize SSH Public Key if that attribute is set - if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(usr, source.loginSource, su.SSHPublicKey) { + if isAttributeSSHPublicKeySet && asymkey_model.SynchronizePublicKeys(usr, source.authSource, su.SSHPublicKey) { sshKeysNeedUpdate = true } @@ -140,7 +140,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { usr.FullName != fullName || !usr.IsActive { - log.Trace("SyncExternalUsers[%s]: Updating user %s", source.loginSource.Name, usr.Name) + log.Trace("SyncExternalUsers[%s]: Updating user %s", source.authSource.Name, usr.Name) usr.FullName = fullName usr.Email = su.Mail @@ -156,7 +156,7 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { err = user_model.UpdateUserCols(db.DefaultContext, usr, "full_name", "email", "is_admin", "is_restricted", "is_active") if err != nil { - log.Error("SyncExternalUsers[%s]: Error updating user %s: %v", source.loginSource.Name, usr.Name, err) + log.Error("SyncExternalUsers[%s]: Error updating user %s: %v", source.authSource.Name, usr.Name, err) } } @@ -179,8 +179,8 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { select { case <-ctx.Done(): - log.Warn("SyncExternalUsers: Cancelled during update of %s before delete users", source.loginSource.Name) - return db.ErrCancelledf("During update of %s before delete users", source.loginSource.Name) + log.Warn("SyncExternalUsers: Cancelled during update of %s before delete users", source.authSource.Name) + return db.ErrCancelledf("During update of %s before delete users", source.authSource.Name) default: } @@ -192,12 +192,12 @@ func (source *Source) Sync(ctx context.Context, updateExisting bool) error { existPos++ } if usr.IsActive && (existPos >= len(existingUsers) || i < existingUsers[existPos]) { - log.Trace("SyncExternalUsers[%s]: Deactivating user %s", source.loginSource.Name, usr.Name) + log.Trace("SyncExternalUsers[%s]: Deactivating user %s", source.authSource.Name, usr.Name) usr.IsActive = false err = user_model.UpdateUserCols(db.DefaultContext, usr, "is_active") if err != nil { - log.Error("SyncExternalUsers[%s]: Error deactivating user %s: %v", source.loginSource.Name, usr.Name, err) + log.Error("SyncExternalUsers[%s]: Error deactivating user %s: %v", source.authSource.Name, usr.Name, err) } } } diff --git a/services/auth/source/oauth2/assert_interface_test.go b/services/auth/source/oauth2/assert_interface_test.go index 0a1986a3b29b..0ec7361ca836 100644 --- a/services/auth/source/oauth2/assert_interface_test.go +++ b/services/auth/source/oauth2/assert_interface_test.go @@ -5,7 +5,7 @@ package oauth2_test import ( - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/oauth2" ) @@ -14,9 +14,9 @@ import ( // It tightly binds the interfaces and implementation without breaking go import cycles type sourceInterface interface { - login.Config - login.SourceSettable - login.RegisterableSource + auth_model.Config + auth_model.SourceSettable + auth_model.RegisterableSource auth.PasswordAuthenticator } diff --git a/services/auth/source/oauth2/init.go b/services/auth/source/oauth2/init.go index edbbb8969ec0..e4eedd34cbdf 100644 --- a/services/auth/source/oauth2/init.go +++ b/services/auth/source/oauth2/init.go @@ -9,7 +9,7 @@ import ( "net/http" "sync" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -52,19 +52,19 @@ func Init() error { // Unlock our mutex gothRWMutex.Unlock() - return initOAuth2LoginSources() + return initOAuth2Sources() } // ResetOAuth2 clears existing OAuth2 providers and loads them from DB func ResetOAuth2() error { ClearProviders() - return initOAuth2LoginSources() + return initOAuth2Sources() } -// initOAuth2LoginSources is used to load and register all active OAuth2 providers -func initOAuth2LoginSources() error { - loginSources, _ := login.GetActiveOAuth2ProviderLoginSources() - for _, source := range loginSources { +// initOAuth2Sources is used to load and register all active OAuth2 providers +func initOAuth2Sources() error { + authSources, _ := auth.GetActiveOAuth2ProviderSources() + for _, source := range authSources { oauth2Source, ok := source.Cfg.(*Source) if !ok { continue diff --git a/services/auth/source/oauth2/providers.go b/services/auth/source/oauth2/providers.go index 18879e917bb8..065c6884e099 100644 --- a/services/auth/source/oauth2/providers.go +++ b/services/auth/source/oauth2/providers.go @@ -9,7 +9,7 @@ import ( "net/url" "sort" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -55,7 +55,7 @@ func NewImagedProvider(image string, provider GothProvider) *ImagedProvider { } // Providers contains the map of registered OAuth2 providers in Gitea (based on goth) -// key is used to map the OAuth2Provider with the goth provider type (also in LoginSource.OAuth2Config.Provider) +// key is used to map the OAuth2Provider with the goth provider type (also in AuthSource.OAuth2Config.Provider) // value is used to store display data var gothProviders = map[string]GothProvider{} @@ -88,14 +88,14 @@ func GetOAuth2Providers() []Provider { func GetActiveOAuth2Providers() ([]string, map[string]Provider, error) { // Maybe also separate used and unused providers so we can force the registration of only 1 active provider for each type - loginSources, err := login.GetActiveOAuth2ProviderLoginSources() + authSources, err := auth.GetActiveOAuth2ProviderSources() if err != nil { return nil, nil, err } var orderedKeys []string providers := make(map[string]Provider) - for _, source := range loginSources { + for _, source := range authSources { prov := gothProviders[source.Cfg.(*Source).Provider] if source.Cfg.(*Source).IconURL != "" { prov = &ImagedProvider{prov, source.Cfg.(*Source).IconURL} @@ -140,8 +140,8 @@ func ClearProviders() { } var ( - // ErrLoginSourceNotActived login source is not actived error - ErrLoginSourceNotActived = errors.New("Login source is not actived") + // ErrAuthSourceNotActived login source is not actived error + ErrAuthSourceNotActived = errors.New("auth source is not actived") ) // used to create different types of goth providers @@ -153,7 +153,7 @@ func createProvider(providerName string, source *Source) (goth.Provider, error) p, ok := gothProviders[source.Provider] if !ok { - return nil, ErrLoginSourceNotActived + return nil, ErrAuthSourceNotActived } provider, err = p.CreateGothProvider(providerName, callbackURL, source) diff --git a/services/auth/source/oauth2/source.go b/services/auth/source/oauth2/source.go index 68ff08d1eeaa..457686ba1fd4 100644 --- a/services/auth/source/oauth2/source.go +++ b/services/auth/source/oauth2/source.go @@ -5,7 +5,7 @@ package oauth2 import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/json" ) @@ -33,8 +33,8 @@ type Source struct { RestrictedGroup string SkipLocalTwoFA bool `json:",omitempty"` - // reference to the loginSource - loginSource *login.Source + // reference to the authSource + authSource *auth.Source } // FromDB fills up an OAuth2Config from serialized format. @@ -47,11 +47,11 @@ func (source *Source) ToDB() ([]byte, error) { return json.Marshal(source) } -// SetLoginSource sets the related LoginSource -func (source *Source) SetLoginSource(loginSource *login.Source) { - source.loginSource = loginSource +// SetAuthSource sets the related AuthSource +func (source *Source) SetAuthSource(authSource *auth.Source) { + source.authSource = authSource } func init() { - login.RegisterTypeConfig(login.OAuth2, &Source{}) + auth.RegisterTypeConfig(auth.OAuth2, &Source{}) } diff --git a/services/auth/source/oauth2/source_callout.go b/services/auth/source/oauth2/source_callout.go index c0ac7e041052..8596dd187d92 100644 --- a/services/auth/source/oauth2/source_callout.go +++ b/services/auth/source/oauth2/source_callout.go @@ -14,7 +14,7 @@ import ( // Callout redirects request/response pair to authenticate against the provider func (source *Source) Callout(request *http.Request, response http.ResponseWriter) error { // not sure if goth is thread safe (?) when using multiple providers - request.Header.Set(ProviderHeaderKey, source.loginSource.Name) + request.Header.Set(ProviderHeaderKey, source.authSource.Name) // don't use the default gothic begin handler to prevent issues when some error occurs // normally the gothic library will write some custom stuff to the response instead of our own nice error page @@ -34,7 +34,7 @@ func (source *Source) Callout(request *http.Request, response http.ResponseWrite // this will trigger a new authentication request, but because we save it in the session we can use that func (source *Source) Callback(request *http.Request, response http.ResponseWriter) (goth.User, error) { // not sure if goth is thread safe (?) when using multiple providers - request.Header.Set(ProviderHeaderKey, source.loginSource.Name) + request.Header.Set(ProviderHeaderKey, source.authSource.Name) gothRWMutex.RLock() defer gothRWMutex.RUnlock() diff --git a/services/auth/source/oauth2/source_register.go b/services/auth/source/oauth2/source_register.go index 71ba288bf4c9..f61de7e1d69c 100644 --- a/services/auth/source/oauth2/source_register.go +++ b/services/auth/source/oauth2/source_register.go @@ -10,13 +10,13 @@ import ( // RegisterSource causes an OAuth2 configuration to be registered func (source *Source) RegisterSource() error { - err := RegisterProviderWithGothic(source.loginSource.Name, source) - return wrapOpenIDConnectInitializeError(err, source.loginSource.Name, source) + err := RegisterProviderWithGothic(source.authSource.Name, source) + return wrapOpenIDConnectInitializeError(err, source.authSource.Name, source) } // UnregisterSource causes an OAuth2 configuration to be unregistered func (source *Source) UnregisterSource() error { - RemoveProviderFromGothic(source.loginSource.Name) + RemoveProviderFromGothic(source.authSource.Name) return nil } diff --git a/services/auth/source/pam/assert_interface_test.go b/services/auth/source/pam/assert_interface_test.go index a151c2f52e6e..d8754cdf77e1 100644 --- a/services/auth/source/pam/assert_interface_test.go +++ b/services/auth/source/pam/assert_interface_test.go @@ -5,7 +5,7 @@ package pam_test import ( - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/pam" ) @@ -15,8 +15,8 @@ import ( type sourceInterface interface { auth.PasswordAuthenticator - login.Config - login.SourceSettable + auth_model.Config + auth_model.SourceSettable } var _ (sourceInterface) = &pam.Source{} diff --git a/services/auth/source/pam/source.go b/services/auth/source/pam/source.go index 65ae76138c8a..957c89dc85e5 100644 --- a/services/auth/source/pam/source.go +++ b/services/auth/source/pam/source.go @@ -5,7 +5,7 @@ package pam import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/json" ) @@ -22,8 +22,8 @@ type Source struct { EmailDomain string SkipLocalTwoFA bool `json:",omitempty"` // Skip Local 2fa for users authenticated with this source - // reference to the loginSource - loginSource *login.Source + // reference to the authSource + authSource *auth.Source } // FromDB fills up a PAMConfig from serialized format. @@ -36,11 +36,11 @@ func (source *Source) ToDB() ([]byte, error) { return json.Marshal(source) } -// SetLoginSource sets the related LoginSource -func (source *Source) SetLoginSource(loginSource *login.Source) { - source.loginSource = loginSource +// SetAuthSource sets the related AuthSource +func (source *Source) SetAuthSource(authSource *auth.Source) { + source.authSource = authSource } func init() { - login.RegisterTypeConfig(login.PAM, &Source{}) + auth.RegisterTypeConfig(auth.PAM, &Source{}) } diff --git a/services/auth/source/pam/source_authenticate.go b/services/auth/source/pam/source_authenticate.go index 8553653ea060..d5bd9409963f 100644 --- a/services/auth/source/pam/source_authenticate.go +++ b/services/auth/source/pam/source_authenticate.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/auth/pam" "code.gitea.io/gitea/modules/setting" @@ -55,8 +55,8 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str Name: username, Email: email, Passwd: password, - LoginType: login.PAM, - LoginSource: source.loginSource.ID, + LoginType: auth.PAM, + LoginSource: source.authSource.ID, LoginName: userName, // This is what the user typed in IsActive: true, } diff --git a/services/auth/source/smtp/assert_interface_test.go b/services/auth/source/smtp/assert_interface_test.go index d1c982472fc6..c7fae6431fa1 100644 --- a/services/auth/source/smtp/assert_interface_test.go +++ b/services/auth/source/smtp/assert_interface_test.go @@ -5,7 +5,7 @@ package smtp_test import ( - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/auth/source/smtp" ) @@ -15,11 +15,11 @@ import ( type sourceInterface interface { auth.PasswordAuthenticator - login.Config - login.SkipVerifiable - login.HasTLSer - login.UseTLSer - login.SourceSettable + auth_model.Config + auth_model.SkipVerifiable + auth_model.HasTLSer + auth_model.UseTLSer + auth_model.SourceSettable } var _ (sourceInterface) = &smtp.Source{} diff --git a/services/auth/source/smtp/source.go b/services/auth/source/smtp/source.go index 006202e0d5ab..5e69f912da35 100644 --- a/services/auth/source/smtp/source.go +++ b/services/auth/source/smtp/source.go @@ -5,7 +5,7 @@ package smtp import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/json" ) @@ -28,8 +28,8 @@ type Source struct { DisableHelo bool SkipLocalTwoFA bool `json:",omitempty"` - // reference to the loginSource - loginSource *login.Source + // reference to the authSource + authSource *auth.Source } // FromDB fills up an SMTPConfig from serialized format. @@ -57,11 +57,11 @@ func (source *Source) UseTLS() bool { return source.ForceSMTPS || source.Port == 465 } -// SetLoginSource sets the related LoginSource -func (source *Source) SetLoginSource(loginSource *login.Source) { - source.loginSource = loginSource +// SetAuthSource sets the related AuthSource +func (source *Source) SetAuthSource(authSource *auth.Source) { + source.authSource = authSource } func init() { - login.RegisterTypeConfig(login.SMTP, &Source{}) + auth.RegisterTypeConfig(auth.SMTP, &Source{}) } diff --git a/services/auth/source/smtp/source_authenticate.go b/services/auth/source/smtp/source_authenticate.go index c32d638b54fa..3be2f1128de2 100644 --- a/services/auth/source/smtp/source_authenticate.go +++ b/services/auth/source/smtp/source_authenticate.go @@ -10,7 +10,7 @@ import ( "net/textproto" "strings" - "code.gitea.io/gitea/models/login" + auth_model "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/mailer" @@ -71,8 +71,8 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str Name: strings.ToLower(username), Email: userName, Passwd: password, - LoginType: login.SMTP, - LoginSource: source.loginSource.ID, + LoginType: auth_model.SMTP, + LoginSource: source.authSource.ID, LoginName: userName, IsActive: true, } diff --git a/services/auth/source/sspi/assert_interface_test.go b/services/auth/source/sspi/assert_interface_test.go index 1efa69c05ba6..33442451867b 100644 --- a/services/auth/source/sspi/assert_interface_test.go +++ b/services/auth/source/sspi/assert_interface_test.go @@ -5,7 +5,7 @@ package sspi_test import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/services/auth/source/sspi" ) @@ -13,7 +13,7 @@ import ( // It tightly binds the interfaces and implementation without breaking go import cycles type sourceInterface interface { - login.Config + auth.Config } var _ (sourceInterface) = &sspi.Source{} diff --git a/services/auth/source/sspi/source.go b/services/auth/source/sspi/source.go index 8d469d481f75..e6e63ee1eb8e 100644 --- a/services/auth/source/sspi/source.go +++ b/services/auth/source/sspi/source.go @@ -5,7 +5,7 @@ package sspi import ( - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/modules/json" ) @@ -36,5 +36,5 @@ func (cfg *Source) ToDB() ([]byte, error) { } func init() { - login.RegisterTypeConfig(login.SSPI, &Source{}) + auth.RegisterTypeConfig(auth.SSPI, &Source{}) } diff --git a/services/auth/sspi_windows.go b/services/auth/sspi_windows.go index 19f234912243..cadf72179646 100644 --- a/services/auth/sspi_windows.go +++ b/services/auth/sspi_windows.go @@ -9,8 +9,8 @@ import ( "net/http" "strings" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/avatars" - "code.gitea.io/gitea/models/login" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/log" @@ -154,7 +154,7 @@ func (s *SSPI) Verify(req *http.Request, w http.ResponseWriter, store DataStore, // getConfig retrieves the SSPI configuration from login sources func (s *SSPI) getConfig() (*sspi.Source, error) { - sources, err := login.ActiveSources(login.SSPI) + sources, err := auth.ActiveSources(auth.SSPI) if err != nil { return nil, err } @@ -250,7 +250,7 @@ func sanitizeUsername(username string, cfg *sspi.Source) string { // fails (or if negotiation should continue), which would prevent other authentication methods // to execute at all. func specialInit() { - if login.IsSSPIEnabled() { + if auth.IsSSPIEnabled() { Register(&SSPI{}) } } diff --git a/services/auth/sync.go b/services/auth/sync.go index 494dfc3a3845..b7f3232a3047 100644 --- a/services/auth/sync.go +++ b/services/auth/sync.go @@ -7,8 +7,8 @@ package auth import ( "context" + "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/models/login" "code.gitea.io/gitea/modules/log" ) @@ -16,7 +16,7 @@ import ( func SyncExternalUsers(ctx context.Context, updateExisting bool) error { log.Trace("Doing: SyncExternalUsers") - ls, err := login.Sources() + ls, err := auth.Sources() if err != nil { log.Error("SyncExternalUsers: %v", err) return err diff --git a/services/externalaccount/user.go b/services/externalaccount/user.go index 8fd0680a1f9d..e5cd443770c0 100644 --- a/services/externalaccount/user.go +++ b/services/externalaccount/user.go @@ -8,7 +8,7 @@ import ( "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/models/login" + "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/structs" @@ -16,14 +16,14 @@ import ( ) func toExternalLoginUser(user *user_model.User, gothUser goth.User) (*user_model.ExternalLoginUser, error) { - loginSource, err := login.GetActiveOAuth2LoginSourceByName(gothUser.Provider) + authSource, err := auth.GetActiveOAuth2SourceByName(gothUser.Provider) if err != nil { return nil, err } return &user_model.ExternalLoginUser{ ExternalID: gothUser.UserID, UserID: user.ID, - LoginSourceID: loginSource.ID, + LoginSourceID: authSource.ID, RawData: gothUser.RawData, Provider: gothUser.Provider, Email: gothUser.Email, diff --git a/templates/user/settings/security_accountlinks.tmpl b/templates/user/settings/security/accountlinks.tmpl similarity index 100% rename from templates/user/settings/security_accountlinks.tmpl rename to templates/user/settings/security/accountlinks.tmpl diff --git a/templates/user/settings/security_openid.tmpl b/templates/user/settings/security/openid.tmpl similarity index 100% rename from templates/user/settings/security_openid.tmpl rename to templates/user/settings/security/openid.tmpl diff --git a/templates/user/settings/security.tmpl b/templates/user/settings/security/security.tmpl similarity index 56% rename from templates/user/settings/security.tmpl rename to templates/user/settings/security/security.tmpl index 891f7b953517..6d5f38ab7076 100644 --- a/templates/user/settings/security.tmpl +++ b/templates/user/settings/security/security.tmpl @@ -3,11 +3,11 @@ {{template "user/settings/navbar" .}}
{{template "base/alert" .}} - {{template "user/settings/security_twofa" .}} - {{template "user/settings/security_u2f" .}} - {{template "user/settings/security_accountlinks" .}} + {{template "user/settings/security/twofa" .}} + {{template "user/settings/security/u2f" .}} + {{template "user/settings/security/accountlinks" .}} {{if .EnableOpenIDSignIn}} - {{template "user/settings/security_openid" .}} + {{template "user/settings/security/openid" .}} {{end}}
diff --git a/templates/user/settings/security_twofa.tmpl b/templates/user/settings/security/twofa.tmpl similarity index 100% rename from templates/user/settings/security_twofa.tmpl rename to templates/user/settings/security/twofa.tmpl diff --git a/templates/user/settings/twofa_enroll.tmpl b/templates/user/settings/security/twofa_enroll.tmpl similarity index 100% rename from templates/user/settings/twofa_enroll.tmpl rename to templates/user/settings/security/twofa_enroll.tmpl diff --git a/templates/user/settings/security_u2f.tmpl b/templates/user/settings/security/u2f.tmpl similarity index 100% rename from templates/user/settings/security_u2f.tmpl rename to templates/user/settings/security/u2f.tmpl From 766cb481f71144a150c48b04197a2396f4174baa Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 2 Jan 2022 17:54:49 +0100 Subject: [PATCH 11/22] Update owners for 2022 (#18155) --- CONTRIBUTING.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b91291127ed8..fad535bb2ac3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -299,6 +299,11 @@ and lead the development of Gitea. To honor the past owners, here's the history of the owners and the time they served: +* 2022-01-01 ~ 2022-12-31 - https://github.com/go-gitea/gitea/issues/17872 + * [Lunny Xiao](https://gitea.com/lunny) + * [Matti Ranta](https://gitea.com/techknowlogick) + * [Andrew Thornton](https://gitea.com/zeripath) + * 2021-01-01 ~ 2021-12-31 - https://github.com/go-gitea/gitea/issues/13801 * [Lunny Xiao](https://gitea.com/lunny) * [Lauris Bukšis-Haberkorns](https://gitea.com/lafriks) From 948949f42989bbbb701a51cfe9d6bd3cd6b246c8 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Mon, 3 Jan 2022 03:30:40 +0800 Subject: [PATCH 12/22] chore(lint): use golangci-lint to call revive and misspell checker. (#18145) replace revive and misspell with golangci-lint Signed-off-by: Bo-Yi Wu --- .golangci.yml | 29 +++++++++++++++++++++++++++++ .revive.toml | 27 --------------------------- Makefile | 29 ++--------------------------- 3 files changed, 31 insertions(+), 54 deletions(-) delete mode 100644 .revive.toml diff --git a/.golangci.yml b/.golangci.yml index 9e64d71b594a..235bb76715bd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,6 +16,7 @@ linters: - gocritic - bidichk - ineffassign + - revive enable-all: false disable-all: true fast: false @@ -28,6 +29,34 @@ linters-settings: disabled-checks: - ifElseChain - singleCaseSwitch # Every time this occurred in the code, there was no other way. + revive: + ignore-generated-header: false + severity: warning + confidence: 0.8 + errorCode: 1 + warningCode: 1 + rules: + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: if-return + - name: increment-decrement + - name: var-naming + - name: var-declaration + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: duplicated-imports + - name: modifies-value-receiver issues: exclude-rules: diff --git a/.revive.toml b/.revive.toml deleted file mode 100644 index 54f075f84f51..000000000000 --- a/.revive.toml +++ /dev/null @@ -1,27 +0,0 @@ -ignoreGeneratedHeader = false -severity = "warning" -confidence = 0.8 -errorCode = 1 -warningCode = 1 - -[rule.blank-imports] -[rule.context-as-argument] -[rule.context-keys-type] -[rule.dot-imports] -[rule.error-return] -[rule.error-strings] -[rule.error-naming] -[rule.exported] -[rule.if-return] -[rule.increment-decrement] -[rule.var-naming] -[rule.var-declaration] -[rule.package-comments] -[rule.range] -[rule.receiver-naming] -[rule.time-naming] -[rule.unexported-return] -[rule.indent-error-flow] -[rule.errorf] -[rule.duplicated-imports] -[rule.modifies-value-receiver] diff --git a/Makefile b/Makefile index 4800bb480696..00318b4afde4 100644 --- a/Makefile +++ b/Makefile @@ -186,8 +186,6 @@ help: @echo " - generate-swagger generate the swagger spec from code comments" @echo " - swagger-validate check if the swagger spec is valid" @echo " - golangci-lint run golangci-lint linter" - @echo " - revive run revive linter" - @echo " - misspell check for misspellings" @echo " - vet examines Go source code and reports suspicious constructs" @echo " - test[\#TestSpecificName] run unit test" @echo " - test-sqlite[\#TestSpecificName] run integration test for sqlite" @@ -280,29 +278,6 @@ errcheck: @echo "Running errcheck..." @errcheck $(GO_PACKAGES) -.PHONY: revive -revive: - @hash revive > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ - $(GO) install github.com/mgechev/revive@v1.1.2; \ - fi - @revive -config .revive.toml -exclude=./vendor/... ./... - -.PHONY: misspell-check -misspell-check: - @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ - $(GO) install github.com/client9/misspell/cmd/misspell@v0.3.4; \ - fi - @echo "Running misspell-check..." - @$(GO) run build/code-batch-process.go misspell -error -i unknwon '{file-list}' - -.PHONY: misspell -misspell: - @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ - $(GO) install github.com/client9/misspell/cmd/misspell@v0.3.4; \ - fi - @echo "Running go misspell..." - @$(GO) run build/code-batch-process.go misspell -w -i unknwon '{file-list}' - .PHONY: fmt-check fmt-check: # get all go files and run go fmt on them @@ -320,7 +295,7 @@ checks: checks-frontend checks-backend checks-frontend: svg-check .PHONY: checks-backend -checks-backend: misspell-check test-vendor swagger-check swagger-validate +checks-backend: test-vendor swagger-check swagger-validate .PHONY: lint lint: lint-frontend lint-backend @@ -332,7 +307,7 @@ lint-frontend: node_modules npx editorconfig-checker templates .PHONY: lint-backend -lint-backend: golangci-lint revive vet +lint-backend: golangci-lint vet .PHONY: watch watch: From 496acbe9e5fb02b6c9ad1a85c328220d4db51a6b Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 2 Jan 2022 22:31:03 +0000 Subject: [PATCH 13/22] Require codereview to have content (#18156) - Report a validityError when the codeReview have no comment. - Resolves #18151 - Refactor --- web_src/js/features/comp/CommentEasyMDE.js | 28 ++++++++++++++++++++++ web_src/js/features/repo-diff.js | 7 ++++++ web_src/js/features/repo-wiki.js | 16 +++---------- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/web_src/js/features/comp/CommentEasyMDE.js b/web_src/js/features/comp/CommentEasyMDE.js index 47e80e69c1cf..8efbe4d34de9 100644 --- a/web_src/js/features/comp/CommentEasyMDE.js +++ b/web_src/js/features/comp/CommentEasyMDE.js @@ -96,3 +96,31 @@ export function getAttachedEasyMDE(el) { } return el._data_easyMDE; } + +/** + * validate if the given textarea from a form, is non-empty. + * @param {jQuery | HTMLElement} form + * @param {jQuery | HTMLElement} textarea + * @returns {boolean} returns true if validation succeeded. + */ +export function validateTextareaNonEmpty(form, textarea) { + if (form instanceof jQuery) { + form = form[0]; + } + if (textarea instanceof jQuery) { + textarea = textarea[0]; + } + + const $markdownEditorTextArea = $(getAttachedEasyMDE(textarea).codemirror.getInputField()); + // The original edit area HTML element is hidden and replaced by the + // SimpleMDE/EasyMDE editor, breaking HTML5 input validation if the text area is empty. + // This is a workaround for this upstream bug. + // See https://github.com/sparksuite/simplemde-markdown-editor/issues/324 + if (textarea.value.length) { + $markdownEditorTextArea.prop('required', true); + form.reportValidity(); + return false; + } + $markdownEditorTextArea.prop('required', false); + return true; +} diff --git a/web_src/js/features/repo-diff.js b/web_src/js/features/repo-diff.js index f8f6c38b765a..d2502315b44b 100644 --- a/web_src/js/features/repo-diff.js +++ b/web_src/js/features/repo-diff.js @@ -1,5 +1,6 @@ import {initCompReactionSelector} from './comp/ReactionSelector.js'; import {initRepoIssueContentHistory} from './repo-issue-content.js'; +import {validateTextareaNonEmpty} from './comp/CommentEasyMDE.js'; const {csrfToken} = window.config; export function initRepoDiffReviewButton() { @@ -23,7 +24,13 @@ export function initRepoDiffFileViewToggle() { export function initRepoDiffConversationForm() { $(document).on('submit', '.conversation-holder form', async (e) => { e.preventDefault(); + const form = $(e.target); + const $textArea = form.find('textarea'); + if (!validateTextareaNonEmpty(form, $textArea)) { + return; + } + const newConversationHolder = $(await $.post(form.attr('action'), form.serialize())); const {path, side, idx} = newConversationHolder.data(); diff --git a/web_src/js/features/repo-wiki.js b/web_src/js/features/repo-wiki.js index ee23dda8c473..c7b902d31df4 100644 --- a/web_src/js/features/repo-wiki.js +++ b/web_src/js/features/repo-wiki.js @@ -1,4 +1,5 @@ import {initMarkupContent} from '../markup/content.js'; +import {validateTextareaNonEmpty} from './comp/CommentEasyMDE.js'; import {initCompMarkupContentPreviewTab} from './comp/MarkupContentPreview.js'; const {csrfToken} = window.config; @@ -121,19 +122,8 @@ export function initRepoWikiForm() { const $markdownEditorTextArea = $(easyMDE.codemirror.getInputField()); $markdownEditorTextArea.addClass('js-quick-submit'); - $form.on('submit', function (e) { - // The original edit area HTML element is hidden and replaced by the - // SimpleMDE/EasyMDE editor, breaking HTML5 input validation if the text area is empty. - // This is a workaround for this upstream bug. - // See https://github.com/sparksuite/simplemde-markdown-editor/issues/324 - const input = $editArea.val(); - if (!input.length) { - e.preventDefault(); - $markdownEditorTextArea.prop('required', true); - this.reportValidity(); - } else { - $markdownEditorTextArea.prop('required', false); - } + $form.on('submit', function () { + validateTextareaNonEmpty(this, $editArea); }); setTimeout(() => { From f499f23039a46a7b0b48ca848ead242fe430da29 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Mon, 3 Jan 2022 00:28:34 +0000 Subject: [PATCH 14/22] [skip ci] Updated translations via Crowdin --- options/locale/locale_zh-CN.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index ff25560f9053..a35263d48c80 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -104,7 +104,12 @@ error404=您正尝试访问的页面 不存在 或 您 never=从不 [error] +occurred=发生了一个错误 +report_message=如果您确定这是一个 Gitea bug，请在 GitHub 上搜索问题，或在必要时打开一个新问题。 missing_csrf=错误的请求：没有 CSRF 令牌 +invalid_csrf=错误的请求：无效的 CSRF 令牌 +not_found=找不到目标。 +network_error=网络错误 [startpage] app_desc=一款极易搭建的自助 Git 服务 From ade41f3f048d71521af2c9df0ea0623c372d48a3 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Mon, 3 Jan 2022 15:35:01 +0000 Subject: [PATCH 15/22] tests: add coverage for models migration helpers (#18162) They were previously not covered at all, either by integration tests or unit tests. This PR also fixes a bug where the `num_comments` field was incorrectly set to include all types of comments. It sets num_closed_issues: 0 as default in milestone unit test fixtures. If they are not set, Incr("num_closed_issues") will be a noop because the field is null. --- models/fixtures/milestone.yml | 5 + models/migrate.go | 2 +- models/migrate_test.go | 167 ++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 models/migrate_test.go diff --git a/models/fixtures/milestone.yml b/models/fixtures/milestone.yml index 2cd994e63a50..d439ce81cb17 100644 --- a/models/fixtures/milestone.yml +++ b/models/fixtures/milestone.yml @@ -5,6 +5,7 @@ content: content1 is_closed: false num_issues: 1 + num_closed_issues: 0 - id: 2 @@ -13,6 +14,7 @@ content: content2 is_closed: false num_issues: 0 + num_closed_issues: 0 - id: 3 @@ -21,6 +23,7 @@ content: content3 is_closed: true num_issues: 1 + num_closed_issues: 0 - id: 4 @@ -29,6 +32,7 @@ content: content random is_closed: false num_issues: 0 + num_closed_issues: 0 - id: 5 @@ -37,3 +41,4 @@ content: for testing with PRs is_closed: false num_issues: 0 + num_closed_issues: 0 diff --git a/models/migrate.go b/models/migrate.go index 07d2b0f2d967..ce529efc4bd5 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -166,7 +166,7 @@ func InsertIssueComments(comments []*Comment) error { } for issueID := range issueIDs { - if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ?) WHERE id = ?", issueID, issueID); err != nil { + if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", issueID, CommentTypeComment, issueID); err != nil { return err } } diff --git a/models/migrate_test.go b/models/migrate_test.go new file mode 100644 index 000000000000..09433b6b4175 --- /dev/null +++ b/models/migrate_test.go @@ -0,0 +1,167 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "testing" + + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + + "github.com/stretchr/testify/assert" +) + +func TestMigrate_InsertMilestones(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + reponame := "repo1" + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository) + name := "milestonetest1" + ms := &Milestone{ + RepoID: repo.ID, + Name: name, + } + err := InsertMilestones(ms) + assert.NoError(t, err) + unittest.AssertExistsAndLoadBean(t, ms) + repoModified := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: repo.ID}).(*repo_model.Repository) + assert.EqualValues(t, repo.NumMilestones+1, repoModified.NumMilestones) + + unittest.CheckConsistencyFor(t, &Milestone{}) +} + +func assertCreateIssues(t *testing.T, reponame string, isPull bool) { + assert.NoError(t, unittest.PrepareTestDatabase()) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository) + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User) + label := unittest.AssertExistsAndLoadBean(t, &Label{ID: 1}).(*Label) + milestone := unittest.AssertExistsAndLoadBean(t, &Milestone{ID: 1}).(*Milestone) + assert.EqualValues(t, milestone.ID, 1) + reaction := &Reaction{ + Type: "heart", + UserID: owner.ID, + } + + title := "issuetitle1" + var is = &Issue{ + RepoID: repo.ID, + MilestoneID: milestone.ID, + Repo: repo, + Title: title, + Content: "issuecontent1", + IsPull: isPull, + PosterID: owner.ID, + Poster: owner, + IsClosed: true, + Labels: []*Label{label}, + Reactions: []*Reaction{reaction}, + } + err := InsertIssues(is) + assert.NoError(t, err) + + i := unittest.AssertExistsAndLoadBean(t, &Issue{Title: title}).(*Issue) + unittest.AssertExistsAndLoadBean(t, &Reaction{Type: "heart", UserID: owner.ID, IssueID: i.ID}) + + labelModified := unittest.AssertExistsAndLoadBean(t, &Label{ID: 1}).(*Label) + assert.EqualValues(t, label.NumIssues+1, labelModified.NumIssues) + assert.EqualValues(t, label.NumClosedIssues+1, labelModified.NumClosedIssues) + + milestoneModified := unittest.AssertExistsAndLoadBean(t, &Milestone{ID: milestone.ID}).(*Milestone) + assert.EqualValues(t, milestone.NumIssues+1, milestoneModified.NumIssues) + assert.EqualValues(t, milestone.NumClosedIssues+1, milestoneModified.NumClosedIssues) +} + +func TestMigrate_CreateIssuesIsPullFalse(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + reponame := "repo1" + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository) + + assertCreateIssues(t, reponame, false) + + repoModified := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: repo.ID}).(*repo_model.Repository) + assert.EqualValues(t, repo.NumIssues+1, repoModified.NumIssues) + assert.EqualValues(t, repo.NumClosedIssues+1, repoModified.NumClosedIssues) +} + +func TestMigrate_CreateIssuesIsPullTrue(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + reponame := "repo1" + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository) + + assertCreateIssues(t, reponame, true) + + repoModified := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: repo.ID}).(*repo_model.Repository) + assert.EqualValues(t, repo.NumPulls+1, repoModified.NumPulls) + assert.EqualValues(t, repo.NumClosedPulls+1, repoModified.NumClosedPulls) +} + +func TestMigrate_InsertIssueComments(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + issue := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 1}).(*Issue) + _ = issue.LoadRepo() + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: issue.Repo.OwnerID}).(*user_model.User) + reaction := &Reaction{ + Type: "heart", + UserID: owner.ID, + } + + comment := &Comment{ + PosterID: owner.ID, + Poster: owner, + IssueID: issue.ID, + Issue: issue, + Reactions: []*Reaction{reaction}, + } + + err := InsertIssueComments([]*Comment{comment}) + assert.NoError(t, err) + + issueModified := unittest.AssertExistsAndLoadBean(t, &Issue{ID: 1}).(*Issue) + assert.EqualValues(t, issue.NumComments+1, issueModified.NumComments) + + unittest.CheckConsistencyFor(t, &Issue{}) +} + +func TestMigrate_InsertPullRequests(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + reponame := "repo1" + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository) + owner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User) + + var i = &Issue{ + RepoID: repo.ID, + Repo: repo, + Title: "title1", + Content: "issuecontent1", + IsPull: true, + PosterID: owner.ID, + Poster: owner, + } + + var p = &PullRequest{ + Issue: i, + } + + err := InsertPullRequests(p) + assert.NoError(t, err) + + _ = unittest.AssertExistsAndLoadBean(t, &PullRequest{IssueID: i.ID}).(*PullRequest) + + unittest.CheckConsistencyFor(t, &Issue{}, &PullRequest{}) +} + +func TestMigrate_InsertReleases(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + a := &repo_model.Attachment{ + UUID: "a0eebc91-9c0c-4ef7-bb6e-6bb9bd380a12", + } + r := &Release{ + Attachments: []*repo_model.Attachment{a}, + } + + err := InsertReleases(r) + assert.NoError(t, err) +} From 8eec403068ef22ba7316fbbbd9254a54847cf5fe Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 4 Jan 2022 00:06:46 +0800 Subject: [PATCH 16/22] Fix bug (#18168) --- templates/admin/dashboard.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin/dashboard.tmpl b/templates/admin/dashboard.tmpl index 79f031882e33..35f45bd3529c 100644 --- a/templates/admin/dashboard.tmpl +++ b/templates/admin/dashboard.tmpl @@ -13,7 +13,7 @@

- {{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.LoginSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}} + {{.i18n.Tr "admin.dashboard.statistic_info" .Stats.Counter.User .Stats.Counter.Org .Stats.Counter.PublicKey .Stats.Counter.Repo .Stats.Counter.Watch .Stats.Counter.Star .Stats.Counter.Action .Stats.Counter.Access .Stats.Counter.Issue .Stats.Counter.Comment .Stats.Counter.Oauth .Stats.Counter.Follow .Stats.Counter.Mirror .Stats.Counter.Release .Stats.Counter.AuthSource .Stats.Counter.Webhook .Stats.Counter.Milestone .Stats.Counter.Label .Stats.Counter.HookTask .Stats.Counter.Team .Stats.Counter.UpdateTask .Stats.Counter.Attachment | Str2html}}

From 48aab263d1d60612be05a67c46172b5a5a718dba Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 4 Jan 2022 00:53:53 +0800 Subject: [PATCH 17/22] Fix EasyMDE validation (#18161) --- web_src/js/features/comp/CommentEasyMDE.js | 48 ++++++++++++++-------- web_src/js/features/repo-diff.js | 2 +- web_src/js/features/repo-wiki.js | 14 ++++--- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/web_src/js/features/comp/CommentEasyMDE.js b/web_src/js/features/comp/CommentEasyMDE.js index 8efbe4d34de9..d3447d7ba237 100644 --- a/web_src/js/features/comp/CommentEasyMDE.js +++ b/web_src/js/features/comp/CommentEasyMDE.js @@ -75,13 +75,27 @@ export function createCommentEasyMDE(textarea) { } }); attachTribute(inputField, {mentions: true, emoji: true}); + attachEasyMDEToElements(easyMDE); + return easyMDE; +} +/** + * attach the EasyMDE object to its input elements (InputField, TextArea) + * @param {EasyMDE} easyMDE + */ +export function attachEasyMDEToElements(easyMDE) { // TODO: that's the only way we can do now to attach the EasyMDE object to a HTMLElement + + // InputField is used by CodeMirror to accept user input + const inputField = easyMDE.codemirror.getInputField(); inputField._data_easyMDE = easyMDE; - textarea._data_easyMDE = easyMDE; - return easyMDE; + + // TextArea is the real textarea element in the form + const textArea = easyMDE.codemirror.getTextArea(); + textArea._data_easyMDE = easyMDE; } + /** * get the attached EasyMDE editor created by createCommentEasyMDE * @param el jQuery or HTMLElement @@ -98,29 +112,27 @@ export function getAttachedEasyMDE(el) { } /** - * validate if the given textarea from a form, is non-empty. - * @param {jQuery | HTMLElement} form - * @param {jQuery | HTMLElement} textarea + * validate if the given EasyMDE textarea is is non-empty. + * @param {jQuery} $textarea * @returns {boolean} returns true if validation succeeded. */ -export function validateTextareaNonEmpty(form, textarea) { - if (form instanceof jQuery) { - form = form[0]; - } - if (textarea instanceof jQuery) { - textarea = textarea[0]; - } - - const $markdownEditorTextArea = $(getAttachedEasyMDE(textarea).codemirror.getInputField()); +export function validateTextareaNonEmpty($textarea) { + const $mdeInputField = $(getAttachedEasyMDE($textarea).codemirror.getInputField()); // The original edit area HTML element is hidden and replaced by the // SimpleMDE/EasyMDE editor, breaking HTML5 input validation if the text area is empty. // This is a workaround for this upstream bug. // See https://github.com/sparksuite/simplemde-markdown-editor/issues/324 - if (textarea.value.length) { - $markdownEditorTextArea.prop('required', true); - form.reportValidity(); + if (!$textarea.val()) { + $mdeInputField.prop('required', true); + const $form = $textarea.parents('form'); + if (!$form.length) { + // this should never happen. we put a alert here in case the textarea would be forgotten to be put in a form + alert('Require non-empty content'); + } else { + $form[0].reportValidity(); + } return false; } - $markdownEditorTextArea.prop('required', false); + $mdeInputField.prop('required', false); return true; } diff --git a/web_src/js/features/repo-diff.js b/web_src/js/features/repo-diff.js index d2502315b44b..3378222ae11a 100644 --- a/web_src/js/features/repo-diff.js +++ b/web_src/js/features/repo-diff.js @@ -27,7 +27,7 @@ export function initRepoDiffConversationForm() { const form = $(e.target); const $textArea = form.find('textarea'); - if (!validateTextareaNonEmpty(form, $textArea)) { + if (!validateTextareaNonEmpty($textArea)) { return; } diff --git a/web_src/js/features/repo-wiki.js b/web_src/js/features/repo-wiki.js index c7b902d31df4..b76dac030e6e 100644 --- a/web_src/js/features/repo-wiki.js +++ b/web_src/js/features/repo-wiki.js @@ -1,5 +1,5 @@ import {initMarkupContent} from '../markup/content.js'; -import {validateTextareaNonEmpty} from './comp/CommentEasyMDE.js'; +import {attachEasyMDEToElements, validateTextareaNonEmpty} from './comp/CommentEasyMDE.js'; import {initCompMarkupContentPreviewTab} from './comp/MarkupContentPreview.js'; const {csrfToken} = window.config; @@ -119,11 +119,15 @@ export function initRepoWikiForm() { ] }); - const $markdownEditorTextArea = $(easyMDE.codemirror.getInputField()); - $markdownEditorTextArea.addClass('js-quick-submit'); + attachEasyMDEToElements(easyMDE); - $form.on('submit', function () { - validateTextareaNonEmpty(this, $editArea); + const $mdeInputField = $(easyMDE.codemirror.getInputField()); + $mdeInputField.addClass('js-quick-submit'); + + $form.on('submit', () => { + if (!validateTextareaNonEmpty($editArea)) { + return false; + } }); setTimeout(() => { From 650a50a7ba8efcd8d02c4c93e5abc19f27e6c4b5 Mon Sep 17 00:00:00 2001 From: Gusted Date: Mon, 3 Jan 2022 17:23:43 +0000 Subject: [PATCH 18/22] Remove unused route "/tasks/trigger" (#18160) ref: https://github.com/go-gitea/gitea/pull/18160#issuecomment-1004091325 --- routers/web/repo/pull.go | 39 ------------------------------------- routers/web/repo/setting.go | 25 ------------------------ routers/web/web.go | 2 -- 3 files changed, 66 deletions(-) diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 1a6a5e55e611..37826abfb4bc 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -7,7 +7,6 @@ package repo import ( - "crypto/subtle" "errors" "fmt" "html" @@ -1208,44 +1207,6 @@ func CompareAndPullRequestPost(ctx *context.Context) { ctx.Redirect(pullIssue.Link()) } -// TriggerTask response for a trigger task request -func TriggerTask(ctx *context.Context) { - pusherID := ctx.FormInt64("pusher") - branch := ctx.FormString("branch") - secret := ctx.FormString("secret") - if len(branch) == 0 || len(secret) == 0 || pusherID <= 0 { - ctx.Error(http.StatusNotFound) - log.Trace("TriggerTask: branch or secret is empty, or pusher ID is not valid") - return - } - owner, repo := parseOwnerAndRepo(ctx) - if ctx.Written() { - return - } - got := []byte(base.EncodeMD5(owner.Salt)) - want := []byte(secret) - if subtle.ConstantTimeCompare(got, want) != 1 { - ctx.Error(http.StatusNotFound) - log.Trace("TriggerTask [%s/%s]: invalid secret", owner.Name, repo.Name) - return - } - - pusher, err := user_model.GetUserByID(pusherID) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.Error(http.StatusNotFound) - } else { - ctx.ServerError("GetUserByID", err) - } - return - } - - log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, "", "") - ctx.Status(202) -} - // CleanUpPullRequest responses for delete merged branch when PR has been merged func CleanUpPullRequest(ctx *context.Context) { issue := checkPullInfo(ctx) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 7ff83cc3d76b..b7269b67186c 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -994,31 +994,6 @@ func DeleteTeam(ctx *context.Context) { }) } -// parseOwnerAndRepo get repos by owner -func parseOwnerAndRepo(ctx *context.Context) (*user_model.User, *repo_model.Repository) { - owner, err := user_model.GetUserByName(ctx.Params(":username")) - if err != nil { - if user_model.IsErrUserNotExist(err) { - ctx.NotFound("GetUserByName", err) - } else { - ctx.ServerError("GetUserByName", err) - } - return nil, nil - } - - repo, err := repo_model.GetRepositoryByName(owner.ID, ctx.Params(":reponame")) - if err != nil { - if repo_model.IsErrRepoNotExist(err) { - ctx.NotFound("GetRepositoryByName", err) - } else { - ctx.ServerError("GetRepositoryByName", err) - } - return nil, nil - } - - return owner, repo -} - // GitHooks hooks of a repository func GitHooks(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings.githooks") diff --git a/routers/web/web.go b/routers/web/web.go index 3589ec9bd663..2796904cee4a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1064,8 +1064,6 @@ func RegisterRoutes(m *web.Route) { m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.pack", repo.GetPackFile) m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.idx", repo.GetIdxFile) }, ignSignInAndCsrf) - - m.Head("/tasks/trigger", repo.TriggerTask) }) }) // ***** END: Repository ***** From 637c3ec5d82f9d727724cc362aea37eeb2f64b40 Mon Sep 17 00:00:00 2001 From: a1012112796 <1012112796@qq.com> Date: Tue, 4 Jan 2022 03:45:58 +0800 Subject: [PATCH 19/22] Don't delete branch if other PRs with this branch are open (#18164) fix #18149 Signed-off-by: a1012112796 <1012112796@qq.com> --- models/pull_list.go | 10 ++++++++++ models/pull_test.go | 12 ++++++++++++ routers/api/v1/repo/pull.go | 11 +++++++++++ routers/web/repo/issue.go | 14 +++++++++++++- routers/web/repo/pull.go | 22 ++++++++++++++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) diff --git a/models/pull_list.go b/models/pull_list.go index 7c5b83ae5f27..9d4d42892883 100644 --- a/models/pull_list.go +++ b/models/pull_list.go @@ -59,6 +59,16 @@ func GetUnmergedPullRequestsByHeadInfo(repoID int64, branch string) ([]*PullRequ Find(&prs) } +// HasUnmergedPullRequestsByHeadInfo checks if there are open and not merged pull request +// by given head information (repo and branch) +func HasUnmergedPullRequestsByHeadInfo(repoID int64, branch string) (bool, error) { + return db.GetEngine(db.DefaultContext). + Where("head_repo_id = ? AND head_branch = ? AND has_merged = ? AND issue.is_closed = ? AND flow = ?", + repoID, branch, false, false, PullRequestFlowGithub). + Join("INNER", "issue", "issue.id = pull_request.issue_id"). + Exist(&PullRequest{}) +} + // GetUnmergedPullRequestsByBaseInfo returns all pull requests that are open and has not been merged // by given base information (repo and branch). func GetUnmergedPullRequestsByBaseInfo(repoID int64, branch string) ([]*PullRequest, error) { diff --git a/models/pull_test.go b/models/pull_test.go index f5e9d486ff79..2567984cc133 100644 --- a/models/pull_test.go +++ b/models/pull_test.go @@ -107,6 +107,18 @@ func TestGetUnmergedPullRequest(t *testing.T) { assert.True(t, IsErrPullRequestNotExist(err)) } +func TestHasUnmergedPullRequestsByHeadInfo(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + exist, err := HasUnmergedPullRequestsByHeadInfo(1, "branch2") + assert.NoError(t, err) + assert.Equal(t, true, exist) + + exist, err = HasUnmergedPullRequestsByHeadInfo(1, "not_exist_branch") + assert.NoError(t, err) + assert.Equal(t, false, exist) +} + func TestGetUnmergedPullRequestsByHeadInfo(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) prs, err := GetUnmergedPullRequestsByHeadInfo(1, "branch2") diff --git a/routers/api/v1/repo/pull.go b/routers/api/v1/repo/pull.go index 2b7280d39be0..8297e35a1776 100644 --- a/routers/api/v1/repo/pull.go +++ b/routers/api/v1/repo/pull.go @@ -873,6 +873,17 @@ func MergePullRequest(ctx *context.APIContext) { log.Trace("Pull request merged: %d", pr.ID) if form.DeleteBranchAfterMerge { + // Don't cleanup when there are other PR's that use this branch as head branch. + exist, err := models.HasUnmergedPullRequestsByHeadInfo(pr.HeadRepoID, pr.HeadBranch) + if err != nil { + ctx.ServerError("HasUnmergedPullRequestsByHeadInfo", err) + return + } + if exist { + ctx.Status(http.StatusOK) + return + } + var headRepo *git.Repository if ctx.Repo != nil && ctx.Repo.Repository != nil && ctx.Repo.Repository.ID == pr.HeadRepoID && ctx.Repo.GitRepo != nil { headRepo = ctx.Repo.GitRepo diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index d55b6c96b6af..ea16de3950b7 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -1600,11 +1600,23 @@ func ViewIssue(ctx *context.Context) { } else { ctx.Data["WontSignReason"] = "not_signed_in" } - ctx.Data["IsPullBranchDeletable"] = canDelete && + + isPullBranchDeletable := canDelete && pull.HeadRepo != nil && git.IsBranchExist(ctx, pull.HeadRepo.RepoPath(), pull.HeadBranch) && (!pull.HasMerged || ctx.Data["HeadBranchCommitID"] == ctx.Data["PullHeadCommitID"]) + if isPullBranchDeletable && pull.HasMerged { + exist, err := models.HasUnmergedPullRequestsByHeadInfo(pull.HeadRepoID, pull.HeadBranch) + if err != nil { + ctx.ServerError("HasUnmergedPullRequestsByHeadInfo", err) + return + } + + isPullBranchDeletable = !exist + } + ctx.Data["IsPullBranchDeletable"] = isPullBranchDeletable + stillCanManualMerge := func() bool { if pull.HasMerged || issue.IsClosed || !ctx.IsSigned { return false diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 37826abfb4bc..b40eb1ea170c 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -1051,6 +1051,17 @@ func MergePullRequest(ctx *context.Context) { log.Trace("Pull request merged: %d", pr.ID) if form.DeleteBranchAfterMerge { + // Don't cleanup when other pr use this branch as head branch + exist, err := models.HasUnmergedPullRequestsByHeadInfo(pr.HeadRepoID, pr.HeadBranch) + if err != nil { + ctx.ServerError("HasUnmergedPullRequestsByHeadInfo", err) + return + } + if exist { + ctx.Redirect(issue.Link()) + return + } + var headRepo *git.Repository if ctx.Repo != nil && ctx.Repo.Repository != nil && pr.HeadRepoID == ctx.Repo.Repository.ID && ctx.Repo.GitRepo != nil { headRepo = ctx.Repo.GitRepo @@ -1222,6 +1233,17 @@ func CleanUpPullRequest(ctx *context.Context) { return } + // Don't cleanup when there are other PR's that use this branch as head branch. + exist, err := models.HasUnmergedPullRequestsByHeadInfo(pr.HeadRepoID, pr.HeadBranch) + if err != nil { + ctx.ServerError("HasUnmergedPullRequestsByHeadInfo", err) + return + } + if exist { + ctx.NotFound("CleanUpPullRequest", nil) + return + } + if err := pr.LoadHeadRepo(); err != nil { ctx.ServerError("LoadHeadRepo", err) return From 74a5d49e866f6432f4ecb29011f0b7e87273b728 Mon Sep 17 00:00:00 2001 From: Gusted Date: Mon, 3 Jan 2022 23:24:12 +0000 Subject: [PATCH 20/22] Include folders into size cost (#18158) - Include folders for the disk consumption size, they should be included as they are also saved on the disk :) - Have a more accurate picture of the size of a repo. - Mostly they are the size of the file system's block size. E.g. 4Kb on Linux. --- modules/util/path.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/util/path.go b/modules/util/path.go index cad50eddc238..f4acf92ba933 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -24,9 +24,9 @@ func EnsureAbsolutePath(path, absoluteBase string) string { return filepath.Join(absoluteBase, path) } -const notRegularFileMode os.FileMode = os.ModeDir | os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular +const notRegularFileMode os.FileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular -// GetDirectorySize returns the dumb disk consumption for a given path +// GetDirectorySize returns the disk consumption for a given path func GetDirectorySize(path string) (int64, error) { var size int64 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { From 0848c716bf68b94b703072fdb07bc282e20e1290 Mon Sep 17 00:00:00 2001 From: GiteaBot Date: Tue, 4 Jan 2022 00:28:34 +0000 Subject: [PATCH 21/22] [skip ci] Updated translations via Crowdin --- options/locale/locale_el-GR.ini | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/options/locale/locale_el-GR.ini b/options/locale/locale_el-GR.ini index 2c210e1e72d6..4e4bb21cc219 100644 --- a/options/locale/locale_el-GR.ini +++ b/options/locale/locale_el-GR.ini @@ -104,7 +104,12 @@ error404=Η σελίδα που προσπαθείτε να φτάσετε εί never=Ποτέ [error] +occurred=Παρουσιάστηκε ένα σφάλμα +report_message=Αν είστε σίγουροι ότι πρόκειται για ένα πρόβλημα στο Gitea, παρακαλώ αναζητήστε στα ζητήματα στο GitHub ή ανοίξτε ένα νέο ζήτημα εάν είναι απαραίτητο. missing_csrf=Bad Request: δεν υπάρχει διακριτικό CSRF +invalid_csrf=Λάθος Αίτημα: μη έγκυρο διακριτικό CSRF +not_found=Ο προορισμός δεν βρέθηκε. +network_error=Σφάλμα δικτύου [startpage] app_desc=Μια ανώδυνη, αυτο-φιλοξενούμενη υπηρεσία Git @@ -121,6 +126,7 @@ license_desc=Κατεβάστε το τεκμηρίωση πριν αλλάξετε τις ρυθμίσεις. +require_db_desc=Το Gitea απαιτεί MySQL, PostgreSQL, MSSQL, SQLite3 ή TiDB (με πρωτόκολλο MySQL). db_title=Ρυθμίσεις Βάσης Δεδομένων db_type=Τύπος της Βάσης Δεδομένων host=Διακομιστής @@ -134,6 +140,11 @@ ssl_mode=SSL charset=Charset path=Διαδρομή sqlite_helper=Διαδρομή αρχείου για τη βάση δεδομένων SQLite3.
Εισάγετε μια απόλυτη διαδρομή αν εκτελείτε το Gitea ως υπηρεσία. +reinstall_error=Προσπαθείτε να εγκαταστήσετε σε μια υπάρχουσα βάση δεδομένων Gitea +reinstall_confirm_message=Η επανεγκατάσταση με μια υπάρχουσα βάση δεδομένων Gitea μπορεί να προκαλέσει πολλαπλά προβλήματα. Στις περισσότερες περιπτώσεις, θα πρέπει να χρησιμοποιήσετε το υπάρχον "app.ini" για να εκτελέσετε το Gitea. Αν γνωρίζετε τι κάνετε, επιβεβαιώστε τα εξής: +reinstall_confirm_check_1=Τα δεδομένα που κρυπτογραφούνται από το SECRET_KEY στο app.ini μπορεί να χαθούν: οι χρήστες ενδέχεται να μην μπορούν να συνδεθούν μέσω 2FA/OTP και η αντιγραφή αποθετηρίων να μην λειτουργεί σωστά. Επιλέγοντας αυτό το κουτί επιβεβαιώνετε ότι το τρέχον αρχείο app.ini περιέχει το σωστό SECRET_KEY. +reinstall_confirm_check_2=Τα αποθετήρια και οι ρυθμίσεις μπορεί να χρειαστεί να επανασυγχρονιστούν. Επιλέγοντας αυτό το κουτί επιβεβαιώνετε ότι θα επανασυγχρονίσετε τα άγκιστρα αποθετηρίων και το αρχείο authorized_keys χειροκίνητα. Επιβεβαιώνετε ότι θα βεβαιωθείτε ότι το αποθετήριο και οι ρυθμίσεις κατόπτρου είναι σωστές. +reinstall_confirm_check_3=Επιβεβαιώνετε ότι είστε απολύτως σίγουροι ότι αυτό το Gitea τρέχει στη σωστή τοποθεσία στο app.ini και ότι είστε σίγουροι ότι θα πρέπει να επανεγκαταστήσετε. Επιβεβαιώνετε ότι αναγνωρίζετε τους παραπάνω κινδύνους. err_empty_db_path=Η διαδρομή της βάσης δεδομένων SQLite3 δεν μπορεί να είναι κενή. no_admin_and_disable_registration=Δεν μπορείτε να απενεργοποιήσετε την ιδιο-εγγραφή χρήστη χωρίς να έχετε δημιουργήσει διαχειριστικό λογαριασμό. err_empty_admin_password=Ο κωδικός πρόσβασης του διαχειριστή δεν μπορεί να είναι κενός. @@ -199,8 +210,12 @@ install_btn_confirm=Εγκατάσταση Gitea test_git_failed=Αδυναμία δοκιμής της εντολής 'git': %v sqlite3_not_available=Αυτή η έκδοση Gitea δεν υποστηρίζει την SQLite3. Παρακαλώ κατεβάστε την επίσημη δυαδική έκδοση από το %s (όχι την έκδοση 'gobuild'). invalid_db_setting=Οι ρυθμίσεις της βάσης δεδομένων δεν είναι έγκυρες: %v +invalid_db_table=Ο πίνακας της βάσης δεδομένων '%s' δεν είναι έγκυρος: %v invalid_repo_path=Η αρχική διαδρομή των αποθετηρίων δεν είναι έγκυρη: %v +invalid_app_data_path=Η διαδρομή δεδομένων εφαρμογής (app data) δεν είναι έγκυρη: %v run_user_not_match=Το όνομα χρήστη 'εκτέλεση ως' δεν είναι το τρέχον όνομα χρήστη: %s -> %s +internal_token_failed=Αποτυχία δημιουργίας εσωτερικού διακριτικού: %v +secret_key_failed=Αποτυχία δημιουργίας μυστικού κλειδιού: %v save_config_failed=Αποτυχία αποθήκευσης ρυθμίσεων: %v invalid_admin_setting=Η ρύθμιση λογαριασμού διαχειριστή δεν είναι έγκυρη: %v install_success=Καλώς ορίσατε! Σας ευχαριστούμε που επιλέξατε το Gitea. Διασκεδάστε το και να είστε καλά! @@ -430,6 +445,7 @@ lang_select_error=Επιλέξτε μια γλώσσα από τη λίστα. username_been_taken=Το όνομα χρήστη χρησιμοποιείται ήδη. username_change_not_local_user=Δεν επιτρέπεται στους μη τοπικούς χρήστες να αλλάξουν το όνομα χρήστη τους. repo_name_been_taken=Το όνομα του αποθετηρίου χρησιμοποιείται ήδη. +repository_force_private=Η επιλογή Μόνο Ιδιωτικά είναι ενεργοποιημένη: τα ιδιωτικά αποθετήρια δεν μπορούν να δημοσιευθούν. repository_files_already_exist=Αρχεία υπάρχουν ήδη για αυτό το αποθετήριο. Επικοινωνήστε με το διαχειριστή του συστήματος. repository_files_already_exist.adopt=Αρχεία υπάρχουν ήδη για αυτό το αποθετήριο και μπορούν να Υιοθετηθούν μόνο. repository_files_already_exist.delete=Τα αρχεία υπάρχουν ήδη για αυτόν το αποθετήριο. Πρέπει να τα διαγράψετε. @@ -615,6 +631,18 @@ gpg_token_help=Μπορείτε να δημιουργήσετε μια υπογ gpg_token_code=echo "%s" | gpg -a --default-key %s --detach-sig gpg_token_signature=Θωρακισμένη υπογραφή GPG key_signature_gpg_placeholder=Ξεκινά με '-----BEGIN PGP SIGNATURE-----' +verify_gpg_key_success=Το κλειδί GPG '%s' έχει επιβεβαιωθεί. +ssh_key_verified=Επαληθευμένο Κλειδί +ssh_key_verified_long=Το κλειδί έχει επαληθευτεί με ένα διακριτικό και μπορεί να χρησιμοποιηθεί για να επαληθεύσει τα commits που ταιριάζουν με οποιεσδήποτε ενεργοποιημένες διευθύνσεις ηλεκτρονικού ταχυδρομείου για αυτόν το χρήστη. +ssh_key_verify=Επαλήθευση +ssh_invalid_token_signature=Το παρεχόμενο κλειδί SSH, υπογραφή ή διακριτικό δεν ταιριάζει ή το διακριτικό έληξε. +ssh_token_required=Πρέπει να δώσετε μια υπογραφή για το παρακάτω διακριτικό +ssh_token=Διακριτικό +ssh_token_help=Μπορείτε να δημιουργήσετε μια υπογραφή χρησιμοποιώντας: +ssh_token_code=echo -n "%s" "ssh-keygen -Y sign -n gitea -f /path_to_your_pubkey +ssh_token_signature=Θωρακισμένη υπογραφή SSH +key_signature_ssh_placeholder=Ξεκινά με '-----BEGIN SSH SIGNATURE-----' +verify_ssh_key_success=Το SSH κλειδί '%s' επαληθεύτηκε. subkeys=Υποκλειδιά key_id=ID Κλειδιού key_name=Όνομα Κλειδιού @@ -771,6 +799,8 @@ visibility_fork_helper=(Αλλάζοντας αυτό θα επηρεάσει ό clone_helper=Χρειάζεστε βοήθεια για τη κλωνοποίηση; Επισκεφθείτε τη Βοήθεια. fork_repo=Δημιουργία ενός Fork fork_from=Fork Από Το +already_forked=Έχετε ήδη κάνει fork το %s +fork_to_different_account=Fork σε διαφορετικό λογαριασμό fork_visibility_helper=Η ορατότητα ενός fork αποθετηρίου δεν μπορεί να αλλάξει. use_template=Χρήση αυτού του πρότυπου clone_in_vsc=Κλωνοποίηση στο VS Code @@ -853,6 +883,7 @@ desc.archived=Αρχειοθετημένο template.items=Αντικείμενα Προτύπου template.git_content=Περιεχόμενο Git (Προεπιλεγμένος Κλάδος) template.git_hooks=Git Hooks +template.git_hooks_tooltip=Δεν μπορείτε να τροποποιήσετε ή να καταργήσετε τα Άγκιστρα Git αφού προστεθούν. Επιλέξτε αυτό μόνο αν εμπιστεύεστε το πρότυπο αποθετήριο. template.webhooks=Webhooks template.topics=Θέματα template.avatar=Εικόνα @@ -890,8 +921,10 @@ migrate_items_releases=Εκδόσεις migrate_repo=Μεταφορά Αποθετηρίου migrate.clone_address=Μεταφορά / Κλωνοποίηση Από Το URL migrate.clone_address_desc=Το HTTP(S) ή Git URL 'κλωνοποίησης' ενός υπάρχοντος αποθετηρίου +migrate.github_token_desc=Μπορείτε να βάλετε ένα ή περισσότερα διακριτικά εδώ, χωρισμένα με κόμμα, για να κάνετε τη μετεγκατάσταση πιο γρήγορα, λόγω του ορίου ρυθμού του GitHub API. ΠΡΟΣΟΧΗ: Η κατάχρηση αυτής της δυνατότητας μπορεί να παραβιάσει την πολιτική του παρόχου υπηρεσιών και να οδηγήσει σε αποκλεισμό του λογαριασμού σας. migrate.clone_local_path=ή μια διαδρομή τοπικού διακομιστή migrate.permission_denied=Δεν επιτρέπεται η εισαγωγή τοπικών αποθετηρίων. +migrate.permission_denied_blocked=Δεν μπορείτε να εισαγάγετε από μη επιτρεπόμενους υπολογιστές, παρακαλούμε ζητήστε από τον διαχειριστή να ελέγξει τις ρυθμίσεις ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS. migrate.invalid_local_path=Η τοπική διαδρομή δεν είναι έγκυρη. Δεν υπάρχει ή δεν είναι φάκελος. migrate.invalid_lfs_endpoint=Η διεύθυνση LFS δεν είναι έγκυρο. migrate.failed=Η μεταφορά απέτυχε: %v @@ -903,11 +936,13 @@ migrate.migrating=Γίνεται μεταφορά από %s... migrate.migrating_failed=Η μετεγρατάσταση από %s απέτυχε. migrate.migrating_failed.error=Σφάλμα: %s migrate.migrating_failed_no_addr=Η μεταφορά απέτυχε. +migrate.github.description=Μεταφορά δεδομένων από το github.com ή άλλους διακομιστές GitHub. migrate.git.description=Μεταφορά μόνο του αποθετηρίου από μια οποιαδήποτε υπηρεσία Git. migrate.gitlab.description=Μεταφορά δεδομένων από το gitlab.com ή άλλες εγκαταστάσεις GitLab. migrate.gitea.description=Μεταφορά δεδομένων από το gitea.com ή άλλες εγκαταστάσεις Gitea. migrate.gogs.description=Μεταφορά δεδομένων από το notabug.org ή άλλες εγκαταστάσεις Gogs. migrate.onedev.description=Μεταφορά δεδομένων από το code.onedev.io ή άλλες εγκαταστάσεις OneDev. +migrate.codebase.description=Μεταφορά δεδομένων από το codebasehq.com. migrate.gitbucket.description=Μεταφορά δεδομένων από διακομιστές GitBucket. migrate.migrating_git=Μεταφορά Δεδομένων Git migrate.migrating_topics=Μεταφορά Θεμάτων @@ -937,6 +972,7 @@ clone_this_repo=Κλωνοποίηση αυτού του αποθετηρίου create_new_repo_command=Δημιουργία νέου αποθετηρίου στη γραμμή εντολών push_exist_repo=Προώθηση ενός υπάρχοντος αποθετηρίου από τη γραμμή εντολών empty_message=Αυτό το αποθετήριο δεν περιέχει τίποτα. +broken_message=Τα δεδομένα Git που διέπουν αυτό το αποθετήριο δεν μπορούν να διαβαστούν. Επικοινωνήστε με το διαχειριστή ή διαγράψτε αυτό το αποθετήριο. code=Κώδικας code.desc=Πρόσβαση στον πηγαίο κώδικα, αρχεία, υποβολές και κλάδους. @@ -961,6 +997,7 @@ release=Κυκλοφορία releases=Κυκλοφορίες tag=Ετικέτα released_this=κυκλοφόρησε αυτό +file.title=%s στο %s file_raw=Ακατέργαστο file_history=Ιστορικό file_view_source=Προβολή Πηγαίου @@ -977,6 +1014,7 @@ commit_graph=Γράφημα Υποβολών commit_graph.select=Επιλογή κλάδων commit_graph.hide_pr_refs=Απόκρυψη Pull Requests commit_graph.monochrome=Μονόχρωμο +commit_graph.color=Έγχρωμο blame=Ευθύνη download_file=Λήψη αρχείου normal_view=Κανονική Προβολή @@ -1030,6 +1068,8 @@ editor.commit_empty_file_text=Το αρχείο που πρόκειται να editor.no_changes_to_show=Δεν υπάρχουν αλλαγές για εμφάνιση. editor.fail_to_update_file=Απέτυχε η ενημέρωση/δημιουργία του αρχείου '%s'. editor.fail_to_update_file_summary=Μήνυμα Σφάλματος: +editor.push_rejected_no_message=Η αλλαγή απορρίφθηκε από το διακομιστή χωρίς κάποιο μήνυμα. Παρακαλώ ελέγξτε τα Άγκιστρα Git. +editor.push_rejected=Η αλλαγή απορρίφθηκε από τον διακομιστή. Παρακαλώ ελέγξτε τα Άγκιστρα Git. editor.push_rejected_summary=Μήνυμα Πλήρους Απόρριψης: editor.add_subdir=Προσθήκη φακέλου… editor.unable_to_upload_files=Αποτυχία αποστολής αρχείων στο '%s' με σφάλμα: %v @@ -1057,6 +1097,7 @@ commits.signed_by=Υπογράφηκε από commits.signed_by_untrusted_user=Υπογράφηκε από μη έμπιστο χρήστη commits.signed_by_untrusted_user_unmatched=Υπογράφηκε από ένα μη έμπιστο χρήστη ο οποίος δεν ταιριάζει με τον υποβολέα commits.gpg_key_id=ID Κλειδιού GPG +commits.ssh_key_fingerprint=Αποτύπωμα Κλειδιού SSH ext_issues=Εξωτ. Ζητήματα ext_issues.desc=Σύνδεση σε εξωτερικό εφαρμογή ζητημάτων. @@ -1092,6 +1133,7 @@ projects.board.set_default=Ορισμός Προεπιλογής projects.board.set_default_desc=Ορίστε αυτόν τον πίνακα ως προεπιλογή για μη κατηγοριοποιημένα ζητήματα και pull requests projects.board.delete=Διαγραφή Πίνακα projects.board.deletion_desc=Η διαγραφή ενός πίνακα έργου μετακινεί όλα τα σχετιζόμενα ζητήματα σε 'Αταξινόμητα'. Συνέχεια; +projects.board.color=Χρώμα projects.open=Άνοιγμα projects.close=Κλείσιμο @@ -1158,6 +1200,9 @@ issues.add_assignee_at=`ανατέθηκε από %s %s` issues.remove_assignee_at=`αφαιρέθηκε η ανάθεση από %s %s` issues.remove_self_assignment=`αφαίρεσαν την ανάθεση τους %s` issues.change_title_at=`άλλαξε το τίτλο από %s σε %s %s` +issues.change_ref_at=`άλλαξε την αναφορά από %s σε %s %s` +issues.remove_ref_at=`αφαίρεση την αναφορά %s %s` +issues.add_ref_at=`πρόσθεσε την αναφορά %s %s` issues.delete_branch_at=`διέγραψε το κλάδο %s %s` issues.open_tab=%d Ανοιχτά issues.close_tab=%d Κλειστά @@ -1195,7 +1240,11 @@ issues.action_milestone_no_select=Χωρίς ορόσημο issues.action_assignee=Αποδέκτης issues.action_assignee_no_select=Κανένας Αποδέκτης issues.opened_by=άνοιξαν %[1]s από %[3]s +pulls.merged_by=από %[3]s συγχωνεύθηκε %[1]s +pulls.merged_by_fake=από %[2]s συγχωνεύθηκε %[1]s +issues.closed_by=από %[3]s έκλεισαν %[1]s issues.opened_by_fake=άνοιξε το %[1]s από %[2]s +issues.closed_by_fake=από %[2]s έκλεισαν %[1]s issues.previous=Προηγούμενο issues.next=Επόμενο issues.open_title=Ανοιχτό @@ -1382,6 +1431,7 @@ compare.compare_head=σύγκριση pulls.desc=Ενεργοποίηση των pull requests και της αξιολόγησης κώδικα. pulls.new=Νέο Pull Request +pulls.view=Προβολή Pull Request pulls.compare_changes=Νέο Pull Request pulls.compare_changes_desc=Επιλέξτε τον κλάδο που θα συγχωνευθεί και τον κλάδο από τον οποίο θα τραβηχτεί. pulls.compare_base=συγχώνευση σε @@ -1459,7 +1509,10 @@ pulls.rebase_conflict_summary=Μήνυμα Σφάλματος ; %[2]s
%[3]s pulls.unrelated_histories=H Συγχώνευση Απέτυχε: Η κεφαλή και η βάση της συγχώνευσης δεν μοιράζονται μια κοινή ιστορία. Συμβουλή: Δοκιμάστε μια διαφορετική στρατηγική pulls.merge_out_of_date=Η συγχώνευση απέτυχε: Κατά τη δημιουργία της συγχώνευσης, η βάση ενημερώθηκε. Συμβουλή: Δοκιμάστε ξανά. +pulls.head_out_of_date=Η συγχώνευση απέτυχε: Κατά τη δημιουργία της συγχώνευσης, το HEAD ενημερώθηκε. Συμβουλή: Δοκιμάστε ξανά. +pulls.push_rejected=Η συγχώνευση απέτυχε: Η ώθηση απορρίφθηκε. Ελέγξτε τα Άγκιστρα Git για αυτό το αποθετήριο. pulls.push_rejected_summary=Μήνυμα Πλήρους Απόρριψης +pulls.push_rejected_no_message=H Συγχώνευση Aπέτυχε: Η ώθηση απορρίφθηκε, αλλά δεν υπήρχε απομακρυσμένο μήνυμα.
Ελέγξτε τα Άγκιστρα Git για αυτό το αποθετήριο pulls.open_unmerged_pull_exists=`Δεν μπορείτε να ανοίξετε εκ νέου, επειδή υπάρχει ένα εκκρεμές pull request (#%d) με πανομοιότυπες ιδιότητες.` pulls.status_checking=Μερικοί έλεγχοι εκκρεμούν pulls.status_checks_success=Όλοι οι έλεγχοι ήταν επιτυχείς @@ -1551,6 +1604,7 @@ wiki.page_already_exists=Υπάρχει ήδη μια σελίδα wiki με τ wiki.reserved_page=Το όνομα σελίδας wiki '%s' είναι δεσμευμένο. wiki.pages=Σελίδες wiki.last_updated=Τελευταία ενημέρωση %s +wiki.page_name_desc=Εισάγετε ένα όνομα για αυτή τη σελίδα Wiki. Μερικά ειδικά ονόματα είναι: 'Home', '_Sidebar' και '_Footer'. activity=Δραστηριότητα activity.period.filter_label=Περίοδος: @@ -1688,6 +1742,12 @@ settings.pulls.default_delete_branch_after_merge=Διαγραφή του κλά settings.projects_desc=Ενεργοποίηση Έργων Αποθετηρίου settings.admin_settings=Ρυθμίσεις Διαχειριστή settings.admin_enable_health_check=Ενεργοποίηση Ελέγχων Υγείας του Αποθετηρίου (git fsck) +settings.admin_code_indexer=Indexer Κώδικα +settings.admin_stats_indexer=Indexer Στατιστικών Κώδικα +settings.admin_indexer_commit_sha=Τελευταίο Indexed SHA +settings.admin_indexer_unindexed=Unindexed +settings.reindex_button=Προσθήκη στην Ουρά Reindex +settings.reindex_requested=Αιτήθηκε Reindex settings.admin_enable_close_issues_via_commit_in_any_branch=Κλείσιμο ενός ζητήματος μέσω μιας υποβολής που έγινε σε έναν μη προεπιλεγμένο κλάδο settings.danger_zone=Επικίνδυνη Περιοχή settings.new_owner_has_same_repo=Ο νέος ιδιοκτήτης έχει ήδη ένα αποθετήριο με το ίδιο όνομα. Παρακαλώ επιλέξτε ένα άλλο όνομα. @@ -1777,6 +1837,7 @@ settings.webhook.response=Απάντηση settings.webhook.headers=Κεφαλίδες settings.webhook.payload=Περιεχόμενο settings.webhook.body=Σώμα +settings.githooks_desc=Τα Άγκιστρα Git παρέχονται από το ίδιο το Git. Μπορείτε να επεξεργαστείτε τα αρχεία αγκίστρων παρακάτω για να ρυθμίσετε προσαρμοσμένες λειτουργίες. settings.githook_edit_desc=Αν το hook είναι ανενεργό, θα παρουσιαστεί ένα παράδειγμα. Αφήνοντας το περιεχόμενο του hook κενό θα το απενεργοποιήσετε. settings.githook_name=Όνομα Hook settings.githook_content=Περιεχόμενο Hook @@ -1788,6 +1849,7 @@ settings.content_type=Τύπος Περιεχομένου POST settings.secret=Μυστικό settings.slack_username=Όνομα Χρήστη settings.slack_icon_url=URL Εικονιδίου +settings.slack_color=Χρώμα settings.discord_username=Όνομα Χρήστη settings.discord_icon_url=URL Εικονιδίου settings.event_desc=Ενεργοποίηση Σε: @@ -2018,6 +2080,7 @@ diff.file_suppressed=Το diff αρχείου καταστέλλεται επε diff.file_suppressed_line_too_long=Το diff αρχείων καταστέλλεται επειδή μία ή περισσότερες γραμμές είναι πολύ μεγάλες diff.too_many_files=Ορισμένα αρχεία δεν εμφανίστηκαν επειδή έχουν αλλάξει πάρα πολλά αρχεία σε αυτή τη διαφορά diff.show_more=Εμφάνιση Περισσότερων +diff.load=Φόρτωση Διαφορών diff.generated=δημιουργημένο diff.vendored=εξωτερικό diff.comment.placeholder=Αφήστε ένα σχόλιο @@ -2480,6 +2543,14 @@ auths.oauth2_emailURL=Email URL auths.skip_local_two_fa=Παράλειψη τοπικού 2FA auths.skip_local_two_fa_helper=Αφήνοντας μη-ορισμένο σημαίνει ότι οι τοπικοί χρήστες με ορισμένο 2FA θα πρέπει να πετύχουν το 2FA για να συνδεθούν auths.oauth2_tenant=Ένοικος +auths.oauth2_scopes=Πρόσθετα Scopes +auths.oauth2_required_claim_name=Απαιτούμενο Όνομα Claim +auths.oauth2_required_claim_name_helper=Ορίστε αυτό το όνομα για να περιορίσετε τη σύνδεση από αυτή την πηγή στους χρήστες με ένα claim με αυτό το όνομα +auths.oauth2_required_claim_value=Απαιτούμενη Τιμή Claim +auths.oauth2_required_claim_value_helper=Ορίστε αυτήν την τιμή για να περιορίσετε τη σύνδεση από αυτή την πηγή στους χρήστες με ένα claim με αυτό το όνομα και την τιμή +auths.oauth2_group_claim_name=Όνομα claim που παρέχει ονόματα ομάδων για αυτήν την πηγή. (Προαιρετικό) +auths.oauth2_admin_group=Τιμή Group Claim για διαχειριστές. (Προαιρετικό - απαιτεί όνομα claim παραπάνω) +auths.oauth2_restricted_group=Τιμή Group Claim για περιορισμένους χρήστες (Προαιρετικό - απαιτεί όνομα claim παραπάνω) auths.enable_auto_register=Ενεργοποίηση Αυτόματης Εγγραφής auths.sspi_auto_create_users=Αυτόματη δημιουργία χρηστών auths.sspi_auto_create_users_helper=Επιτρέψτε στη μέθοδο πιστοποίησης SSPI να δημιουργεί αυτόματα νέους λογαριασμούς για χρήστες που συνδέονται για πρώτη φορά @@ -2669,6 +2740,7 @@ monitor.execute_time=Χρόνος Εκτέλεσης monitor.process.cancel=Ακύρωση διαδικασίας monitor.process.cancel_desc=Η ακύρωση μιας διαδικασίας μπορεί να προκαλέσει απώλεια δεδομένων monitor.process.cancel_notices=Ακύρωση: %s; +monitor.process.children=Θυγατρικές monitor.queues=Ουρές monitor.queue=Ουρά: %s monitor.queue.name=Όνομα From 165346c15c6d021028a65121e692a17ffc927e2c Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 4 Jan 2022 03:36:47 +0000 Subject: [PATCH 22/22] Add MP4 as default allowed attachment type (#18170) --- custom/conf/app.example.ini | 2 +- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 2 +- modules/setting/attachment.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 1d19a343804f..d86092350021 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1617,7 +1617,7 @@ PATH = ;ENABLED = true ;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. -;ALLOWED_TYPES = .docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip +;ALLOWED_TYPES = .docx,.gif,.gz,.jpeg,.jpg,.mp4,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip ;; ;; Max size of each file. Defaults to 4MB ;MAX_SIZE = 4 diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 90117efb24de..02a61fe44576 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -731,7 +731,7 @@ Default templates for project boards: ## Issue and pull request attachments (`attachment`) - `ENABLED`: **true**: Whether issue and pull request attachments are enabled. -- `ALLOWED_TYPES`: **.docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip**: Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. +- `ALLOWED_TYPES`: **.docx,.gif,.gz,.jpeg,.jpg,mp4,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip**: Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. - `MAX_SIZE`: **4**: Maximum size (MB). - `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once. - `STORAGE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]` diff --git a/modules/setting/attachment.go b/modules/setting/attachment.go index 98c4be94e9cd..409c2b87e54c 100644 --- a/modules/setting/attachment.go +++ b/modules/setting/attachment.go @@ -29,7 +29,7 @@ func newAttachmentService() { Attachment.Storage = getStorage("attachments", storageType, sec) - Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip") + Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.mp4,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip") Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4) Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5) Attachment.Enabled = sec.Key("ENABLED").MustBool(true)