Skip to content

Commit 6222cee

Browse files
andycheng123andycheng123
authored
Merge pull request #785 from andycheng123/improve/optimizer-progressbar
optimizer: add progressbar
2 parents 6b6686c + a029509 commit 6222cee

File tree

4 files changed

+36
-11
lines changed

4 files changed

+36
-11
lines changed

‎go.mod

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ require (
1010
github.com/adshao/go-binance/v2 v2.3.5
1111
github.com/c9s/requestgen v1.3.0
1212
github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b
13+
github.com/cheggaaa/pb/v3 v3.0.8
1314
github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482
1415
github.com/evanphx/json-patch/v5 v5.6.0
1516
github.com/fatih/camelcase v1.0.0
@@ -55,6 +56,7 @@ require (
5556
)
5657

5758
require (
59+
github.com/VividCortex/ewma v1.1.1 // indirect
5860
github.com/beorn7/perks v1.0.1 // indirect
5961
github.com/bitly/go-simplejson v0.5.0 // indirect
6062
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
@@ -86,19 +88,19 @@ require (
8688
github.com/magiconair/properties v1.8.4 // indirect
8789
github.com/mattn/go-colorable v0.1.9 // indirect
8890
github.com/mattn/go-isatty v0.0.14 // indirect
91+
github.com/mattn/go-runewidth v0.0.12 // indirect
8992
github.com/mattn/go-sqlite3 v1.14.13 // indirect
9093
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
9194
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
9295
github.com/mitchellh/mapstructure v1.4.1 // indirect
9396
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
9497
github.com/modern-go/reflect2 v1.0.1 // indirect
95-
github.com/muesli/clusters v0.0.0-20180605185049-a07a36e67d36 // indirect
96-
github.com/muesli/kmeans v0.3.0 // indirect
9798
github.com/pelletier/go-toml v1.8.1 // indirect
9899
github.com/pmezard/go-difflib v1.0.0 // indirect
99100
github.com/prometheus/client_model v0.2.0 // indirect
100101
github.com/prometheus/common v0.32.1 // indirect
101102
github.com/prometheus/procfs v0.7.3 // indirect
103+
github.com/rivo/uniseg v0.2.0 // indirect
102104
github.com/russross/blackfriday/v2 v2.0.1 // indirect
103105
github.com/shopspring/decimal v1.2.0 // indirect
104106
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect

‎go.sum

+12
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
4343
github.com/Masterminds/squirrel v1.5.3 h1:YPpoceAcxuzIljlr5iWpNKaql7hLeG1KLSrhvdHpkZc=
4444
github.com/Masterminds/squirrel v1.5.3/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
4545
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
46+
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
47+
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
4648
github.com/adshao/go-binance/v2 v2.3.5 h1:WVYZecm0w8l14YoWlnKZj6xxZT2AKMTHpMQSqIX1xxA=
4749
github.com/adshao/go-binance/v2 v2.3.5/go.mod h1:8Pg/FGTLyAhq8QXA0IkoReKyRpoxJcK3LVujKDAZV/c=
4850
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -77,6 +79,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
7779
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
7880
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
7981
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
82+
github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
83+
github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
8084
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
8185
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
8286
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -122,6 +126,7 @@ github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0
122126
github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
123127
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
124128
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
129+
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
125130
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
126131
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
127132
github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
@@ -334,13 +339,16 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
334339
github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY=
335340
github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
336341
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
342+
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
337343
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
338344
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
339345
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
340346
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
341347
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
342348
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
343349
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
350+
github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
351+
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
344352
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
345353
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
346354
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -434,6 +442,9 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
434442
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
435443
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
436444
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
445+
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
446+
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
447+
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
437448
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
438449
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
439450
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -698,6 +709,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
698709
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
699710
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
700711
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
712+
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
701713
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
702714
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
703715
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

‎pkg/optimizer/grid.go

+12-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"github.com/cheggaaa/pb/v3"
78
"sort"
89

910
"github.com/evanphx/json-patch/v5"
@@ -172,7 +173,7 @@ func (o *GridOptimizer) Run(executor Executor, configJson []byte) (map[string][]
172173

173174
var ops = o.buildOps()
174175

175-
var taskC = make(chan BacktestTask, 100)
176+
var taskC = make(chan BacktestTask, 10000)
176177

177178
var app = func(configJson []byte, next func(configJson []byte) error) error {
178179
var labels = copyLabels(o.ParamLabels)
@@ -200,27 +201,33 @@ func (o *GridOptimizer) Run(executor Executor, configJson []byte) (map[string][]
200201
}
201202

202203
ctx := context.Background()
203-
resultsC, err := executor.Run(ctx, taskC)
204-
if err != nil {
204+
if err := wrapper(configJson); err != nil {
205205
return nil, err
206206
}
207207

208-
if err := wrapper(configJson); err != nil {
208+
bar := pb.Full.Start(len(taskC))
209+
bar.SetTemplateString(`{{ string . "log" | green}} | {{counters . }} {{bar . }} {{percent . }} {{etime . }} {{rtime . "ETA %s"}}`)
210+
211+
resultsC, err := executor.Run(ctx, taskC, bar)
212+
if err != nil {
209213
return nil, err
210214
}
215+
211216
close(taskC) // this will shut down the executor
212217

213218
for result := range resultsC {
214219
for metricName, metricFunc := range valueFunctions {
215220
var metricValue = metricFunc(result.Report)
216-
log.Infof("params: %+v => %s %+v", result.Params, metricName, metricValue)
221+
bar.Set("log", fmt.Sprintf("params: %+v => %s %+v", result.Params, metricName, metricValue))
222+
bar.Increment()
217223
metrics[metricName] = append(metrics[metricName], Metric{
218224
Params: result.Params,
219225
Labels: result.Labels,
220226
Value: metricValue,
221227
})
222228
}
223229
}
230+
bar.Finish()
224231

225232
for n := range metrics {
226233
sort.Slice(metrics[n], func(i, j int) bool {

‎pkg/optimizer/local.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package optimizer
33
import (
44
"context"
55
"encoding/json"
6+
"fmt"
7+
"github.com/cheggaaa/pb/v3"
68
"os"
79
"os/exec"
810
"strings"
@@ -26,7 +28,7 @@ type BacktestTask struct {
2628

2729
type Executor interface {
2830
// Execute(configJson []byte) (*backtest.SummaryReport, error)
29-
Run(ctx context.Context, taskC chan BacktestTask) (chan BacktestTask, error)
31+
Run(ctx context.Context, taskC chan BacktestTask, bar *pb.ProgressBar) (chan BacktestTask, error)
3032
}
3133

3234
type AsyncHandle struct {
@@ -73,7 +75,7 @@ func (e *LocalProcessExecutor) readReport(output []byte) (*backtest.SummaryRepor
7375
return summaryReport, nil
7476
}
7577

76-
func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask) (chan BacktestTask, error) {
78+
func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask, bar *pb.ProgressBar) (chan BacktestTask, error) {
7779
var maxNumOfProcess = e.Config.MaxNumberOfProcesses
7880
var resultsC = make(chan BacktestTask, maxNumOfProcess*2)
7981

@@ -89,7 +91,8 @@ func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask)
8991
// fork workers
9092
go func(id int, taskC chan BacktestTask) {
9193
taskCnt := 0
92-
log.Infof("starting local worker #%d", id)
94+
bar.Set("log", fmt.Sprintf("starting local worker #%d", id))
95+
bar.Write()
9396
defer wg.Done()
9497
for {
9598
select {
@@ -102,7 +105,8 @@ func (e *LocalProcessExecutor) Run(ctx context.Context, taskC chan BacktestTask)
102105
}
103106

104107
taskCnt++
105-
log.Infof("local worker #%d received param task: %v", id, task.Params)
108+
bar.Set("log", fmt.Sprintf("local worker #%d received param task: %v", id, task.Params))
109+
bar.Write()
106110

107111
report, err := e.execute(task.ConfigJson)
108112
if err != nil {

0 commit comments

Comments
 (0)