Skip to content

Commit

Permalink
add basic retry for http requests (#350)
Browse files Browse the repository at this point in the history 
Signed-off-by: Manabu McCloskey <[email protected]>
  • Loading branch information
@nabuskey
nabuskey authored Jul 31, 2024
1 parent 62afe1f commit f82ac2c
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 32 deletions.
26 changes: 13 additions & 13 deletions tests/e2e/docker/docker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ func CleanUpDocker(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
b, err := e2e.RunCommand(ctx, `docker ps -aqf name=localdev-control-plane`, 10*time.Second)
assert.Nil(t, err, fmt.Sprintf("error while listing docker containers: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while listing docker containers: %s, %s", err, b))

conts := strings.TrimSpace(string(b))
if len(conts) == 0 {
return
}
b, err = e2e.RunCommand(ctx, fmt.Sprintf("docker container rm -f %s", conts), 60*time.Second)
assert.Nil(t, err, fmt.Sprintf("error while removing docker containers: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while removing docker containers: %s, %s", err, b))

b, err = e2e.RunCommand(ctx, "docker system prune -f", 60*time.Second)
assert.Nil(t, err, fmt.Sprintf("error while pruning system: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while pruning system: %s, %s", err, b))

b, err = e2e.RunCommand(ctx, "docker volume prune -f", 60*time.Second)
assert.Nil(t, err, fmt.Sprintf("error while pruning volumes: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while pruning volumes: %s, %s", err, b))
t.Log("finished cleaning up docker env")
}

Expand All @@ -59,10 +59,10 @@ func testCreate(t *testing.T) {
t.Log("running idpbuilder create")
cmd := exec.CommandContext(ctx, e2e.IdpbuilderBinaryLocation, "create")
b, err := cmd.CombinedOutput()
assert.Nil(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))
assert.NoError(t, err, b)

kubeClient, err := e2e.GetKubeClient()
assert.Nil(t, err, fmt.Sprintf("error while getting client: %s", err))
assert.NoError(t, err, fmt.Sprintf("error while getting client: %s", err))

e2e.TestArgoCDApps(ctx, t, kubeClient, e2e.CorePackages)

Expand All @@ -80,10 +80,10 @@ func testCreatePath(t *testing.T) {
t.Log("running idpbuilder create --use-path-routing")
cmd := exec.CommandContext(ctx, e2e.IdpbuilderBinaryLocation, "create", "--use-path-routing")
b, err := cmd.CombinedOutput()
assert.Nil(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))

kubeClient, err := e2e.GetKubeClient()
assert.Nil(t, err, fmt.Sprintf("error while getting client: %s", err))
assert.NoError(t, err, fmt.Sprintf("error while getting client: %s", err))

e2e.TestArgoCDApps(ctx, t, kubeClient, e2e.CorePackages)

Expand All @@ -101,10 +101,10 @@ func testCreatePort(t *testing.T) {
t.Logf("running idpbuilder create --port %s", port)
cmd := exec.CommandContext(ctx, e2e.IdpbuilderBinaryLocation, "create", "--port", port)
b, err := cmd.CombinedOutput()
assert.Nil(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))

kubeClient, err := e2e.GetKubeClient()
assert.Nil(t, err, fmt.Sprintf("error while getting client: %s", err))
assert.NoError(t, err, fmt.Sprintf("error while getting client: %s", err))

e2e.TestArgoCDApps(ctx, t, kubeClient, e2e.CorePackages)

Expand All @@ -123,11 +123,11 @@ func testCustomPkg(t *testing.T) {
t.Log(fmt.Sprintf("running %s", cmdString))
cmd := exec.CommandContext(ctx, e2e.IdpbuilderBinaryLocation, strings.Split(cmdString, " ")...)
b, err := cmd.CombinedOutput()
assert.Nil(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))
assert.NoError(t, err, fmt.Sprintf("error while running create: %s, %s", err, b))

kubeClient, err := e2e.GetKubeClient()

assert.Nil(t, err, fmt.Sprintf("error while getting client: %s", err))
assert.NoError(t, err, fmt.Sprintf("error while getting client: %s", err))
if err != nil {
assert.FailNow(t, "failed creating cluster")
}
Expand All @@ -142,7 +142,7 @@ func testCustomPkg(t *testing.T) {
}
e2e.TestArgoCDApps(ctx, t, kubeClient, expectedApps)
repos, err := e2e.GetGiteaRepos(ctx, giteaBaseUrl)
assert.Nil(t, err)
assert.NoError(t, err)
expectedRepoNames := map[string]struct{}{
"idpbuilder-localdev-my-app-app1": {},
"idpbuilder-localdev-my-app2-app2": {},
Expand Down
69 changes: 50 additions & 19 deletions tests/e2e/e2e.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ const (
GiteaSessionEndpoint = "/api/v1/users/%s/tokens"
GiteaUserEndpoint = "/api/v1/users/%s"
GiteaRepoEndpoint = "/api/v1/repos/search"

httpRetryDelay = 5 * time.Second
httpRetryTimeout = 300 * time.Second
)

var (
Expand Down Expand Up @@ -97,24 +100,52 @@ func RunCommand(ctx context.Context, command string, timeout time.Duration) ([]b
return b, nil
}

func SendAndParse(target any, httpClient *http.Client, req *http.Request) error {
resp, err := httpClient.Do(req)
if err != nil {
return fmt.Errorf("running http request: %w", err)
func SendAndParse(ctx context.Context, target any, httpClient *http.Client, req *http.Request) error {
sendCtx, cancel := context.WithTimeout(ctx, httpRetryTimeout)
defer cancel()
var bodyBytes []byte
if req.Body != nil {
b, bErr := io.ReadAll(req.Body)
if bErr != nil {
return fmt.Errorf("failed copying http request body: %w", bErr)
}
bodyBytes = b
}

defer resp.Body.Close()
for {
select {
case <-sendCtx.Done():
return fmt.Errorf("timedout")
default:
if req.Body != nil {
b := append(make([]byte, 0, len(bodyBytes)), bodyBytes...)
req.Body = io.NopCloser(bytes.NewBuffer(b))
}
resp, err := httpClient.Do(req)
if err != nil {
fmt.Println("failed running http request: ", err)
time.Sleep(httpRetryDelay)
continue
}

defer resp.Body.Close()

respB, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("reading http response body: %w", err)
}
respB, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Println("failed reading http response body: ", err)
time.Sleep(httpRetryDelay)
continue
}

err = json.Unmarshal(respB, target)
if err != nil {
return fmt.Errorf("parsing response body %s, %s", err, respB)
err = json.Unmarshal(respB, target)
if err != nil {
fmt.Println("failed parsing response body: ", err, "\n", string(respB))
time.Sleep(httpRetryDelay)
continue
}
return nil
}
}
return nil
}

func TestGiteaEndpoints(ctx context.Context, t *testing.T, baseUrl string) {
Expand Down Expand Up @@ -159,15 +190,15 @@ func GetGiteaRepos(ctx context.Context, baseUrl string) ([]gitea.Repository, err
req.Header.Set("Authorization", fmt.Sprintf("token %s", token))

user := gitea.User{}
err = SendAndParse(&user, httpClient, req)
err = SendAndParse(ctx, &user, httpClient, req)
if err != nil {
return nil, fmt.Errorf("getting user info %w", err)
}

repos := GiteaSearchRepoResponse{}
repoEp := fmt.Sprintf("%s%s", baseUrl, GiteaRepoEndpoint)
repoReq, _ := http.NewRequestWithContext(ctx, http.MethodGet, repoEp, nil)
err = SendAndParse(&repos, httpClient, repoReq)
err = SendAndParse(ctx, &repos, httpClient, repoReq)
if err != nil {
return nil, fmt.Errorf("getting gitea repositories %w", err)
}
Expand All @@ -189,7 +220,7 @@ func GetGiteaSessionToken(ctx context.Context, auth BasicAuth, baseUrl string) (
sessionReq.Header.Set("Content-Type", "application/json")

var sess gitea.AccessToken
err = SendAndParse(&sess, httpClient, sessionReq)
err = SendAndParse(ctx, &sess, httpClient, sessionReq)
if err != nil {
return "", err
}
Expand All @@ -214,7 +245,7 @@ func TestArgoCDEndpoints(ctx context.Context, t *testing.T, baseUrl string) {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))

var appResp ArgoCDAppResp
err = SendAndParse(&appResp, httpClient, req)
err = SendAndParse(ctx, &appResp, httpClient, req)
assert.Nil(t, err, fmt.Sprintf("getting argocd applications: %s", err))

assert.Equal(t, 3, len(appResp.Items), fmt.Sprintf("number of apps do not match: %v", appResp.Items))
Expand Down Expand Up @@ -267,7 +298,7 @@ func GetArgoCDSessionToken(ctx context.Context, endpoint string) (string, error)
req.Header.Set("Content-Type", "application/json")

var tokenResp ArgoCDAuthResponse
err = SendAndParse(&tokenResp, httpClient, req)
err = SendAndParse(ctx, &tokenResp, httpClient, req)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -301,7 +332,7 @@ func TestArgoCDApps(ctx context.Context, t *testing.T, kubeClient client.Client,
}
if len(apps) != 0 {
t.Logf("waiting for apps to be ready")
time.Sleep(3 * time.Second)
time.Sleep(httpRetryDelay)
continue
}
done = true
Expand Down

0 comments on commit f82ac2c

Please sign in to comment.