forked from go-gitea/gitea
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[GITEA] Enable mocked HTTP responses for GitLab migration test
Fix gitlab migration unit test Closes go-gitea#1837. The differences in dates can be explained by commit e19b965, which changed the order in which "created_date" and "updated_date" are considered. (cherry picked from commit b0bba20aa44e30ef0296b89f336d426224d73a16) Mock HTTP requests in GitLab migration test This introduces a new utility which can be added to other tests making HTTP calls to a live service, to cache the responses of this service in the repository. (cherry picked from commit 52053b138948bd74c7eb88c0796c2e18f4247f3c) Enable mocked HTTP responses for GitLab migration test (cherry picked from commit 19cefc4de24b935a6a5c92be8360301f196f3aa5) Simplify HTTP mocking utility in unit tests Follow-up to https://codeberg.org/forgejo/forgejo/pulls/1841 (cherry picked from commit ca517c8bb4bf97f061b8b19fd3303d734f46660c) (cherry picked from commit b227e0dd6bdf2dc3e8679443fc538fbce4b3bcf5) (cherry picked from commit 6cc9d06556cda6c952a0542284fbe504114971ce) (cherry picked from commit f0746e648dc30510d655b8a3b821199b2638800f)
- Loading branch information
1 parent
72c84eb
commit 4141933
Showing
24 changed files
with
708 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
// Copyright 2017 The Forgejo Authors. All rights reserved. | ||
// SPDX-License-Identifier: MIT | ||
|
||
package unittest | ||
|
||
import ( | ||
"bufio" | ||
"fmt" | ||
"io" | ||
"net/http" | ||
"net/http/httptest" | ||
"net/url" | ||
"os" | ||
"slices" | ||
"strings" | ||
"testing" | ||
|
||
"code.gitea.io/gitea/modules/log" | ||
|
||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
// Mocks HTTP responses of a third-party service (such as GitHub, GitLab…) | ||
// This has two modes: | ||
// - live mode: the requests made to the mock HTTP server are transmitted to the live | ||
// service, and responses are saved as test data files | ||
// - test mode: the responses to requests to the mock HTTP server are read from the | ||
// test data files | ||
func NewMockWebServer(t *testing.T, liveServerBaseURL, testDataDir string, liveMode bool) *httptest.Server { | ||
mockServerBaseURL := "" | ||
ignoredHeaders := []string{"cf-ray", "server", "date", "report-to", "nel", "x-request-id"} | ||
|
||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
path := NormalizedFullPath(r.URL) | ||
log.Info("Mock HTTP Server: got request for path %s", r.URL.Path) | ||
// TODO check request method (support POST?) | ||
fixturePath := fmt.Sprintf("%s/%s", testDataDir, strings.ReplaceAll(path, "/", "_")) | ||
if liveMode { | ||
liveURL := fmt.Sprintf("%s%s", liveServerBaseURL, path) | ||
|
||
request, err := http.NewRequest(r.Method, liveURL, nil) | ||
assert.NoError(t, err, "constructing an HTTP request to %s failed", liveURL) | ||
for headerName, headerValues := range r.Header { | ||
// do not pass on the encoding: let the Transport of the HTTP client handle that for us | ||
if strings.ToLower(headerName) != "accept-encoding" { | ||
for _, headerValue := range headerValues { | ||
request.Header.Add(headerName, headerValue) | ||
} | ||
} | ||
} | ||
|
||
response, err := http.DefaultClient.Do(request) | ||
assert.NoError(t, err, "HTTP request to %s failed: %s", liveURL) | ||
|
||
fixture, err := os.Create(fixturePath) | ||
assert.NoError(t, err, "failed to open the fixture file %s for writing", fixturePath) | ||
defer fixture.Close() | ||
fixtureWriter := bufio.NewWriter(fixture) | ||
|
||
for headerName, headerValues := range response.Header { | ||
for _, headerValue := range headerValues { | ||
if !slices.Contains(ignoredHeaders, strings.ToLower(headerName)) { | ||
_, err := fixtureWriter.WriteString(fmt.Sprintf("%s: %s\n", headerName, headerValue)) | ||
assert.NoError(t, err, "writing the header of the HTTP response to the fixture file failed") | ||
} | ||
} | ||
} | ||
_, err = fixtureWriter.WriteString("\n") | ||
assert.NoError(t, err, "writing the header of the HTTP response to the fixture file failed") | ||
fixtureWriter.Flush() | ||
|
||
log.Info("Mock HTTP Server: writing response to %s", fixturePath) | ||
_, err = io.Copy(fixture, response.Body) | ||
assert.NoError(t, err, "writing the body of the HTTP response to %s failed", liveURL) | ||
|
||
err = fixture.Sync() | ||
assert.NoError(t, err, "writing the body of the HTTP response to the fixture file failed") | ||
} | ||
|
||
fixture, err := os.ReadFile(fixturePath) | ||
assert.NoError(t, err, "missing mock HTTP response: "+fixturePath) | ||
|
||
w.WriteHeader(http.StatusOK) | ||
|
||
// replace any mention of the live HTTP service by the mocked host | ||
stringFixture := strings.ReplaceAll(string(fixture), liveServerBaseURL, mockServerBaseURL) | ||
// parse back the fixture file into a series of HTTP headers followed by response body | ||
lines := strings.Split(stringFixture, "\n") | ||
for idx, line := range lines { | ||
colonIndex := strings.Index(line, ": ") | ||
if colonIndex != -1 { | ||
w.Header().Set(line[0:colonIndex], line[colonIndex+2:]) | ||
} else { | ||
// we reached the end of the headers (empty line), so what follows is the body | ||
responseBody := strings.Join(lines[idx+1:], "\n") | ||
_, err := w.Write([]byte(responseBody)) | ||
assert.NoError(t, err, "writing the body of the HTTP response failed") | ||
break | ||
} | ||
} | ||
})) | ||
mockServerBaseURL = server.URL | ||
return server | ||
} | ||
|
||
func NormalizedFullPath(url *url.URL) string { | ||
// TODO normalize path (remove trailing slash?) | ||
// TODO normalize RawQuery (order query parameters?) | ||
if len(url.Query()) == 0 { | ||
return url.EscapedPath() | ||
} | ||
return fmt.Sprintf("%s?%s", url.EscapedPath(), url.RawQuery) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
services/migrations/testdata/gitlab/full_download/_api_v4_projects_15578026
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
X-Frame-Options: SAMEORIGIN | ||
Ratelimit-Resettime: Thu, 30 Nov 2023 08:45:46 GMT | ||
Cf-Cache-Status: MISS | ||
Set-Cookie: _cfuvid=TkY5Br2q4C67LJ2jZWlgdQaosj3Z4aI81Qb27PNKXfo-1701333886606-0-604800000; path=/; domain=.gitlab.com; HttpOnly; Secure; SameSite=None | ||
Etag: W/"3cacfe29f44a69e84a577337eac55d89" | ||
X-Content-Type-Options: nosniff | ||
Ratelimit-Remaining: 1996 | ||
Gitlab-Lb: haproxy-main-29-lb-gprd | ||
Cache-Control: max-age=0, private, must-revalidate | ||
Referrer-Policy: strict-origin-when-cross-origin | ||
X-Gitlab-Meta: {"correlation_id":"291f87cd975a51a7b756806ce7b53e2e","version":"1"} | ||
Ratelimit-Observed: 4 | ||
Vary: Origin, Accept-Encoding | ||
Gitlab-Sv: localhost | ||
Content-Security-Policy: default-src 'none' | ||
Ratelimit-Reset: 1701333946 | ||
Content-Type: application/json | ||
X-Runtime: 0.101081 | ||
Strict-Transport-Security: max-age=31536000 | ||
Ratelimit-Limit: 2000 | ||
|
||
{"id":15578026,"description":"Test repository for testing migration from gitlab to gitea","name":"test_repo","name_with_namespace":"gitea / test_repo","path":"test_repo","path_with_namespace":"gitea/test_repo","created_at":"2019-11-28T08:20:33.019Z","default_branch":"master","tag_list":["migration","test"],"topics":["migration","test"],"ssh_url_to_repo":"[email protected]:gitea/test_repo.git","http_url_to_repo":"https://gitlab.com/gitea/test_repo.git","web_url":"https://gitlab.com/gitea/test_repo","readme_url":"https://gitlab.com/gitea/test_repo/-/blob/master/README.md","forks_count":1,"avatar_url":null,"star_count":0,"last_activity_at":"2020-04-19T19:46:04.527Z","namespace":{"id":3181312,"name":"gitea","path":"gitea","kind":"group","full_path":"gitea","parent_id":null,"avatar_url":"/uploads/-/system/group/avatar/3181312/gitea.png","web_url":"https://gitlab.com/groups/gitea"},"container_registry_image_prefix":"registry.gitlab.com/gitea/test_repo","_links":{"self":"https://gitlab.com/api/v4/projects/15578026","issues":"https://gitlab.com/api/v4/projects/15578026/issues","merge_requests":"https://gitlab.com/api/v4/projects/15578026/merge_requests","repo_branches":"https://gitlab.com/api/v4/projects/15578026/repository/branches","labels":"https://gitlab.com/api/v4/projects/15578026/labels","events":"https://gitlab.com/api/v4/projects/15578026/events","members":"https://gitlab.com/api/v4/projects/15578026/members","cluster_agents":"https://gitlab.com/api/v4/projects/15578026/cluster_agents"},"packages_enabled":true,"empty_repo":false,"archived":false,"visibility":"public","resolve_outdated_diff_discussions":false,"issues_enabled":true,"merge_requests_enabled":true,"wiki_enabled":true,"jobs_enabled":true,"snippets_enabled":true,"container_registry_enabled":true,"service_desk_enabled":true,"can_create_merge_request_in":true,"issues_access_level":"enabled","repository_access_level":"enabled","merge_requests_access_level":"enabled","forking_access_level":"enabled","wiki_access_level":"enabled","builds_access_level":"enabled","snippets_access_level":"enabled","pages_access_level":"enabled","analytics_access_level":"enabled","container_registry_access_level":"enabled","security_and_compliance_access_level":"private","releases_access_level":"enabled","environments_access_level":"enabled","feature_flags_access_level":"enabled","infrastructure_access_level":"enabled","monitor_access_level":"enabled","model_experiments_access_level":"enabled","emails_disabled":false,"emails_enabled":true,"shared_runners_enabled":true,"lfs_enabled":true,"creator_id":1241334,"import_status":"none","open_issues_count":0,"description_html":"\u003cp data-sourcepos=\"1:1-1:58\" dir=\"auto\"\u003eTest repository for testing migration from gitlab to gitea\u003c/p\u003e","updated_at":"2022-08-26T19:41:46.691Z","ci_config_path":null,"public_jobs":true,"shared_with_groups":[],"only_allow_merge_if_pipeline_succeeds":false,"allow_merge_on_skipped_pipeline":null,"request_access_enabled":true,"only_allow_merge_if_all_discussions_are_resolved":false,"remove_source_branch_after_merge":true,"printing_merge_request_link_enabled":true,"merge_method":"ff","squash_option":"default_off","enforce_auth_checks_on_uploads":true,"suggestion_commit_message":null,"merge_commit_template":null,"squash_commit_template":null,"issue_branch_template":null,"autoclose_referenced_issues":true,"external_authorization_classification_label":"","requirements_enabled":false,"requirements_access_level":"enabled","security_and_compliance_enabled":false,"compliance_frameworks":[],"permissions":{"project_access":null,"group_access":null}} |
29 changes: 29 additions & 0 deletions
29
...itlab/full_download/_api_v4_projects_15578026_issues?page=1&per_page=2&sort=asc&state=all
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
Cache-Control: max-age=0, private, must-revalidate | ||
Vary: Origin, Accept-Encoding | ||
Gitlab-Lb: haproxy-main-24-lb-gprd | ||
Content-Type: application/json | ||
X-Per-Page: 2 | ||
X-Prev-Page: | ||
Set-Cookie: _cfuvid=N.nfy5eIdFH5lXhsnMyEbeBkoxabcl1SVeyyP0_NrdE-1701333887790-0-604800000; path=/; domain=.gitlab.com; HttpOnly; Secure; SameSite=None | ||
Content-Security-Policy: default-src 'none' | ||
X-Content-Type-Options: nosniff | ||
X-Next-Page: | ||
X-Total-Pages: 1 | ||
Etag: W/"4c0531a3595f741f229f5a105e013b95" | ||
X-Gitlab-Meta: {"correlation_id":"b2eca136986f016d946685fb99287f1c","version":"1"} | ||
Ratelimit-Observed: 8 | ||
Ratelimit-Resettime: Thu, 30 Nov 2023 08:45:47 GMT | ||
X-Frame-Options: SAMEORIGIN | ||
Referrer-Policy: strict-origin-when-cross-origin | ||
Ratelimit-Remaining: 1992 | ||
Gitlab-Sv: localhost | ||
Cf-Cache-Status: MISS | ||
Strict-Transport-Security: max-age=31536000 | ||
Ratelimit-Limit: 2000 | ||
X-Total: 2 | ||
X-Page: 1 | ||
X-Runtime: 0.178587 | ||
Ratelimit-Reset: 1701333947 | ||
Link: <https://gitlab.com/api/v4/projects/15578026/issues?id=15578026&order_by=created_at&page=1&per_page=2&sort=asc&state=all&with_labels_details=false>; rel="first", <https://gitlab.com/api/v4/projects/15578026/issues?id=15578026&order_by=created_at&page=1&per_page=2&sort=asc&state=all&with_labels_details=false>; rel="last" | ||
|
||
[{"id":27687675,"iid":1,"project_id":15578026,"title":"Please add an animated gif icon to the merge button","description":"I just want the merge button to hurt my eyes a little. :stuck_out_tongue_closed_eyes:","state":"closed","created_at":"2019-11-28T08:43:35.459Z","updated_at":"2019-11-28T08:46:23.304Z","closed_at":"2019-11-28T08:46:23.275Z","closed_by":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"labels":["bug","discussion"],"milestone":{"id":1082926,"iid":1,"project_id":15578026,"title":"1.0.0","description":"","state":"closed","created_at":"2019-11-28T08:42:30.301Z","updated_at":"2019-11-28T15:57:52.401Z","due_date":null,"start_date":null,"expired":false,"web_url":"https://gitlab.com/gitea/test_repo/-/milestones/1"},"assignees":[],"author":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"type":"ISSUE","assignee":null,"user_notes_count":0,"merge_requests_count":0,"upvotes":1,"downvotes":0,"due_date":null,"confidential":false,"discussion_locked":null,"issue_type":"issue","web_url":"https://gitlab.com/gitea/test_repo/-/issues/1","time_stats":{"time_estimate":0,"total_time_spent":0,"human_time_estimate":null,"human_total_time_spent":null},"task_completion_status":{"count":0,"completed_count":0},"blocking_issues_count":0,"has_tasks":true,"task_status":"0 of 0 checklist items completed","_links":{"self":"https://gitlab.com/api/v4/projects/15578026/issues/1","notes":"https://gitlab.com/api/v4/projects/15578026/issues/1/notes","award_emoji":"https://gitlab.com/api/v4/projects/15578026/issues/1/award_emoji","project":"https://gitlab.com/api/v4/projects/15578026","closed_as_duplicate_of":null},"references":{"short":"#1","relative":"#1","full":"gitea/test_repo#1"},"severity":"UNKNOWN","moved_to_id":null,"service_desk_reply_to":null},{"id":27687706,"iid":2,"project_id":15578026,"title":"Test issue","description":"This is test issue 2, do not touch!","state":"closed","created_at":"2019-11-28T08:44:46.277Z","updated_at":"2019-11-28T08:45:44.987Z","closed_at":"2019-11-28T08:45:44.959Z","closed_by":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"labels":["duplicate"],"milestone":{"id":1082927,"iid":2,"project_id":15578026,"title":"1.1.0","description":"","state":"active","created_at":"2019-11-28T08:42:44.575Z","updated_at":"2019-11-28T08:42:44.575Z","due_date":null,"start_date":null,"expired":false,"web_url":"https://gitlab.com/gitea/test_repo/-/milestones/2"},"assignees":[],"author":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"type":"ISSUE","assignee":null,"user_notes_count":2,"merge_requests_count":0,"upvotes":1,"downvotes":1,"due_date":null,"confidential":false,"discussion_locked":null,"issue_type":"issue","web_url":"https://gitlab.com/gitea/test_repo/-/issues/2","time_stats":{"time_estimate":0,"total_time_spent":0,"human_time_estimate":null,"human_total_time_spent":null},"task_completion_status":{"count":0,"completed_count":0},"blocking_issues_count":0,"has_tasks":true,"task_status":"0 of 0 checklist items completed","_links":{"self":"https://gitlab.com/api/v4/projects/15578026/issues/2","notes":"https://gitlab.com/api/v4/projects/15578026/issues/2/notes","award_emoji":"https://gitlab.com/api/v4/projects/15578026/issues/2/award_emoji","project":"https://gitlab.com/api/v4/projects/15578026","closed_as_duplicate_of":null},"references":{"short":"#2","relative":"#2","full":"gitea/test_repo#2"},"severity":"UNKNOWN","moved_to_id":null,"service_desk_reply_to":null}] |
29 changes: 29 additions & 0 deletions
29
...ata/gitlab/full_download/_api_v4_projects_15578026_issues_1_award_emoji?page=1&per_page=2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
X-Total-Pages: 1 | ||
Referrer-Policy: strict-origin-when-cross-origin | ||
Cache-Control: max-age=0, private, must-revalidate | ||
X-Frame-Options: SAMEORIGIN | ||
X-Runtime: 0.052748 | ||
X-Total: 2 | ||
Gitlab-Sv: localhost | ||
Link: <https://gitlab.com/api/v4/projects/15578026/issues/1/award_emoji?id=15578026&issue_iid=1&page=1&per_page=2>; rel="first", <https://gitlab.com/api/v4/projects/15578026/issues/1/award_emoji?id=15578026&issue_iid=1&page=1&per_page=2>; rel="last" | ||
X-Content-Type-Options: nosniff | ||
X-Gitlab-Meta: {"correlation_id":"22fc215ac386644c9cb8736b652cf702","version":"1"} | ||
X-Per-Page: 2 | ||
Strict-Transport-Security: max-age=31536000 | ||
Ratelimit-Reset: 1701333947 | ||
Content-Security-Policy: default-src 'none' | ||
Ratelimit-Remaining: 1991 | ||
Ratelimit-Observed: 9 | ||
X-Next-Page: | ||
Ratelimit-Resettime: Thu, 30 Nov 2023 08:45:47 GMT | ||
Ratelimit-Limit: 2000 | ||
Gitlab-Lb: haproxy-main-43-lb-gprd | ||
Cf-Cache-Status: MISS | ||
Etag: W/"69c922434ed11248c864d157eb8eabfc" | ||
Content-Type: application/json | ||
Vary: Origin, Accept-Encoding | ||
X-Page: 1 | ||
X-Prev-Page: | ||
Set-Cookie: _cfuvid=POpffkskz4lvLcv2Fhjp7lF3MsmIOugWDzFGtb3ZUig-1701333887995-0-604800000; path=/; domain=.gitlab.com; HttpOnly; Secure; SameSite=None | ||
|
||
[{"id":3009580,"name":"thumbsup","user":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"created_at":"2019-11-28T08:43:40.322Z","updated_at":"2019-11-28T08:43:40.322Z","awardable_id":27687675,"awardable_type":"Issue","url":null},{"id":3009585,"name":"open_mouth","user":{"id":1241334,"username":"lafriks","name":"Lauris BH","state":"active","locked":false,"avatar_url":"https://gitlab.com/uploads/-/system/user/avatar/1241334/avatar.png","web_url":"https://gitlab.com/lafriks"},"created_at":"2019-11-28T08:44:01.902Z","updated_at":"2019-11-28T08:44:01.902Z","awardable_id":27687675,"awardable_type":"Issue","url":null}] |
Oops, something went wrong.