|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "encoding/json" |
| 5 | + "fmt" |
| 6 | + "log" |
| 7 | + "strings" |
| 8 | + "sync" |
| 9 | + "sync/atomic" |
| 10 | + "time" |
| 11 | + |
| 12 | + "github.com/bfirsh/funker-go" |
| 13 | + "github.com/docker/docker/hack/integration-cli-on-swarm/agent/types" |
| 14 | +) |
| 15 | + |
| 16 | +const ( |
| 17 | + // funkerRetryTimeout is for the issue https://github.com/bfirsh/funker/issues/3 |
| 18 | + // When all the funker replicas are busy in their own job, we cannot connect to funker. |
| 19 | + funkerRetryTimeout = 1 * time.Hour |
| 20 | + funkerRetryDuration = 1 * time.Second |
| 21 | +) |
| 22 | + |
| 23 | +// ticker is needed for some CI (e.g., on Travis, job is aborted when no output emitted for 10 minutes) |
| 24 | +func ticker(d time.Duration) chan struct{} { |
| 25 | + t := time.NewTicker(d) |
| 26 | + stop := make(chan struct{}) |
| 27 | + go func() { |
| 28 | + for { |
| 29 | + select { |
| 30 | + case <-t.C: |
| 31 | + log.Printf("tick (just for keeping CI job active) per %s", d.String()) |
| 32 | + case <-stop: |
| 33 | + t.Stop() |
| 34 | + } |
| 35 | + } |
| 36 | + }() |
| 37 | + return stop |
| 38 | +} |
| 39 | + |
| 40 | +func executeTests(funkerName string, testChunks [][]string) error { |
| 41 | + tickerStopper := ticker(9*time.Minute + 55*time.Second) |
| 42 | + defer func() { |
| 43 | + close(tickerStopper) |
| 44 | + }() |
| 45 | + begin := time.Now() |
| 46 | + log.Printf("Executing %d chunks in parallel, against %q", len(testChunks), funkerName) |
| 47 | + var wg sync.WaitGroup |
| 48 | + var passed, failed uint32 |
| 49 | + for chunkID, tests := range testChunks { |
| 50 | + log.Printf("Executing chunk %d (contains %d test filters)", chunkID, len(tests)) |
| 51 | + wg.Add(1) |
| 52 | + go func(chunkID int, tests []string) { |
| 53 | + defer wg.Done() |
| 54 | + chunkBegin := time.Now() |
| 55 | + result, err := executeTestChunkWithRetry(funkerName, types.Args{ |
| 56 | + ChunkID: chunkID, |
| 57 | + Tests: tests, |
| 58 | + }) |
| 59 | + if result.RawLog != "" { |
| 60 | + for _, s := range strings.Split(result.RawLog, "\n") { |
| 61 | + log.Printf("Log (chunk %d): %s", chunkID, s) |
| 62 | + } |
| 63 | + } |
| 64 | + if err != nil { |
| 65 | + log.Printf("Error while executing chunk %d: %v", |
| 66 | + chunkID, err) |
| 67 | + atomic.AddUint32(&failed, 1) |
| 68 | + } else { |
| 69 | + if result.Code == 0 { |
| 70 | + atomic.AddUint32(&passed, 1) |
| 71 | + } else { |
| 72 | + atomic.AddUint32(&failed, 1) |
| 73 | + } |
| 74 | + log.Printf("Finished chunk %d [%d/%d] with %d test filters in %s, code=%d.", |
| 75 | + chunkID, passed+failed, len(testChunks), len(tests), |
| 76 | + time.Now().Sub(chunkBegin), result.Code) |
| 77 | + } |
| 78 | + }(chunkID, tests) |
| 79 | + } |
| 80 | + wg.Wait() |
| 81 | + // TODO: print actual tests rather than chunks |
| 82 | + log.Printf("Executed %d chunks in %s. PASS: %d, FAIL: %d.", |
| 83 | + len(testChunks), time.Now().Sub(begin), passed, failed) |
| 84 | + if failed > 0 { |
| 85 | + return fmt.Errorf("%d chunks failed", failed) |
| 86 | + } |
| 87 | + return nil |
| 88 | +} |
| 89 | + |
| 90 | +func executeTestChunk(funkerName string, args types.Args) (types.Result, error) { |
| 91 | + ret, err := funker.Call(funkerName, args) |
| 92 | + if err != nil { |
| 93 | + return types.Result{}, err |
| 94 | + } |
| 95 | + tmp, err := json.Marshal(ret) |
| 96 | + if err != nil { |
| 97 | + return types.Result{}, err |
| 98 | + } |
| 99 | + var result types.Result |
| 100 | + err = json.Unmarshal(tmp, &result) |
| 101 | + return result, err |
| 102 | +} |
| 103 | + |
| 104 | +func executeTestChunkWithRetry(funkerName string, args types.Args) (types.Result, error) { |
| 105 | + begin := time.Now() |
| 106 | + for i := 0; time.Now().Sub(begin) < funkerRetryTimeout; i++ { |
| 107 | + result, err := executeTestChunk(funkerName, args) |
| 108 | + if err == nil { |
| 109 | + log.Printf("executeTestChunk(%q, %d) returned code %d in trial %d", funkerName, args.ChunkID, result.Code, i) |
| 110 | + return result, nil |
| 111 | + } |
| 112 | + if errorSeemsInteresting(err) { |
| 113 | + log.Printf("Error while calling executeTestChunk(%q, %d), will retry (trial %d): %v", |
| 114 | + funkerName, args.ChunkID, i, err) |
| 115 | + } |
| 116 | + // TODO: non-constant sleep |
| 117 | + time.Sleep(funkerRetryDuration) |
| 118 | + } |
| 119 | + return types.Result{}, fmt.Errorf("could not call executeTestChunk(%q, %d) in %v", funkerName, args.ChunkID, funkerRetryTimeout) |
| 120 | +} |
| 121 | + |
| 122 | +// errorSeemsInteresting returns true if err does not seem about https://github.com/bfirsh/funker/issues/3 |
| 123 | +func errorSeemsInteresting(err error) bool { |
| 124 | + boringSubstrs := []string{"connection refused", "connection reset by peer", "no such host", "transport endpoint is not connected", "no route to host"} |
| 125 | + errS := err.Error() |
| 126 | + for _, boringS := range boringSubstrs { |
| 127 | + if strings.Contains(errS, boringS) { |
| 128 | + return false |
| 129 | + } |
| 130 | + } |
| 131 | + return true |
| 132 | +} |
0 commit comments