diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 3e6e34ea36d2..0e60ddc90da9 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -43,6 +43,10 @@ "Comment": "release-105", "Rev": "98c78185197025f935947caac56a7b6d022f89d2" }, + { + "ImportPath": "github.com/AaronO/go-git-http", + "Rev": "0ebecedc64b67a3a8674c56724082660be48216e" + }, { "ImportPath": "github.com/AdRoll/goamz/aws", "Rev": "c73835dc8fc6958baf8df8656864ee4d6d04b130" diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/.gitignore b/Godeps/_workspace/src/github.com/AaronO/go-git-http/.gitignore new file mode 100644 index 000000000000..836562412fe8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/LICENSE b/Godeps/_workspace/src/github.com/AaronO/go-git-http/LICENSE new file mode 100644 index 000000000000..5c304d1a4a7b --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/README.md b/Godeps/_workspace/src/github.com/AaronO/go-git-http/README.md new file mode 100644 index 000000000000..c74d11ff441e --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/README.md @@ -0,0 +1,77 @@ +go-git-http +=========== + +A Smart Git Http server library in Go (golang) + +### Example + +```go +package main + +import ( + "log" + "net/http" + + "github.com/AaronO/go-git-http" +) + +func main() { + // Get git handler to serve a directory of repos + git := githttp.New("/Users/aaron/git") + + // Attach handler to http server + http.Handle("/", git) + + // Start HTTP server + err := http.ListenAndServe(":8080", nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} +``` + +### Authentication example + +```go +package main + +import ( + "log" + "net/http" + + "github.com/AaronO/go-git-http" + "github.com/AaronO/go-git-http/auth" +) + + +func main() { + // Get git handler to serve a directory of repos + git := githttp.New("/Users/aaron/git") + + // Build an authentication middleware based on a function + authenticator := auth.Authenticator(func(info auth.AuthInfo) (bool, error) { + // Disallow Pushes (making git server pull only) + if info.Push { + return false, nil + } + + // Typically this would be a database lookup + if info.Username == "admin" && info.Password == "password" { + return true, nil + } + + return false, nil + }) + + // Attach handler to http server + // wrap authenticator around git handler + http.Handle("/", authenticator(git)) + + // Start HTTP server + err := http.ListenAndServe(":8080", nil) + if err != nil { + log.Fatal("ListenAndServe: ", err) + } +} +``` + diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth.go new file mode 100644 index 000000000000..6df0aef36888 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth.go @@ -0,0 +1,106 @@ +package auth + +import ( + "net/http" + "regexp" + "strings" +) + +type AuthInfo struct { + // Usernane or email + Username string + // Plaintext password or token + Password string + + // repo component of URL + // Usually: "username/repo_name" + // But could also be: "some_repo.git" + Repo string + + // Are we pushing or fetching ? + Push bool + Fetch bool +} + +var ( + repoNameRegex = regexp.MustCompile("^/?(.*?)/(HEAD|git-upload-pack|git-receive-pack|info/refs|objects/.*)$") +) + +func Authenticator(authf func(AuthInfo) (bool, error)) func(http.Handler) http.Handler { + return func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + auth, err := parseAuthHeader(req.Header.Get("Authorization")) + if err != nil { + w.Header().Set("WWW-Authenticate", `Basic realm="git server"`) + http.Error(w, err.Error(), 401) + return + } + + // Build up info from request headers and URL + info := AuthInfo{ + Username: auth.Name, + Password: auth.Pass, + Repo: repoName(req.URL.Path), + Push: isPush(req), + Fetch: isFetch(req), + } + + // Call authentication function + authenticated, err := authf(info) + if err != nil { + code := 500 + msg := err.Error() + if se, ok := err.(StatusError); ok { + code = se.StatusCode() + } + http.Error(w, msg, code) + return + } + + // Deny access to repo + if !authenticated { + http.Error(w, "Forbidden", 403) + return + } + + // Access granted + handler.ServeHTTP(w, req) + }) + } +} + +func isFetch(req *http.Request) bool { + return isService("upload-pack", req) +} + +func isPush(req *http.Request) bool { + return isService("receive-pack", req) +} + +func isService(service string, req *http.Request) bool { + return getServiceType(req) == service || strings.HasSuffix(req.URL.Path, service) +} + +func repoName(urlPath string) string { + matches := repoNameRegex.FindStringSubmatch(urlPath) + if matches == nil { + return "" + } + return matches[1] +} + +func getServiceType(r *http.Request) string { + service_type := r.FormValue("service") + + if s := strings.HasPrefix(service_type, "git-"); !s { + return "" + } + + return strings.Replace(service_type, "git-", "", 1) +} + +// StatusCode is an interface allowing authenticators +// to pass down error's with an http error code +type StatusError interface { + StatusCode() int +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth_test.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth_test.go new file mode 100644 index 000000000000..f3b3510d0b82 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/auth_test.go @@ -0,0 +1,15 @@ +package auth + +import ( + "testing" +) + +func TestRepoName(t *testing.T) { + if x := repoName("/yapp.ss.git/HEAD"); x != "yapp.ss.git" { + t.Errorf("Should have been 'yapp.js.git' is '%s'", x) + } + + if x := repoName("aarono/gogo-proxy/HEAD"); x != "aarono/gogo-proxy" { + t.Errorf("Should have been 'aarono/gogo-proxy' is '%s'", x) + } +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth.go new file mode 100644 index 000000000000..44593a92586e --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth.go @@ -0,0 +1,47 @@ +package auth + +import ( + "encoding/base64" + "fmt" + "regexp" + "strings" +) + +// Parse http basic header +type BasicAuth struct { + Name string + Pass string +} + +var ( + basicAuthRegex = regexp.MustCompile("^([^:]*):(.*)$") +) + +func parseAuthHeader(header string) (*BasicAuth, error) { + parts := strings.SplitN(header, " ", 2) + if len(parts) < 2 { + return nil, fmt.Errorf("Invalid authorization header, not enought parts") + } + + authType := parts[0] + authData := parts[1] + + if strings.ToLower(authType) != "basic" { + return nil, fmt.Errorf("Authentication '%s' was not of 'Basic' type", authType) + } + + data, err := base64.StdEncoding.DecodeString(authData) + if err != nil { + return nil, err + } + + matches := basicAuthRegex.FindStringSubmatch(string(data)) + if matches == nil { + return nil, fmt.Errorf("Authorization data '%s' did not match auth regexp", data) + } + + return &BasicAuth{ + Name: matches[1], + Pass: matches[2], + }, nil +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth_test.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth_test.go new file mode 100644 index 000000000000..256d6d330987 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/auth/basicauth_test.go @@ -0,0 +1,28 @@ +package auth + +import ( + "testing" +) + +func TestHeaderParsing(t *testing.T) { + // Basic admin:password + authorization := "Basic YWRtaW46cGFzc3dvcmQ=" + + auth, err := parseAuthHeader(authorization) + if err != nil { + t.Error(err) + } + + if auth.Name != "admin" { + t.Errorf("Detected name does not match: '%s'", auth.Name) + } + if auth.Pass != "password" { + t.Errorf("Detected password does not match: '%s'", auth.Pass) + } +} + +func TestEmptyHeader(t *testing.T) { + if _, err := parseAuthHeader(""); err == nil { + t.Errorf("Empty headers should generate errors") + } +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/errors.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/errors.go new file mode 100644 index 000000000000..4a56db26dd96 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/errors.go @@ -0,0 +1,14 @@ +package githttp + +import ( + "fmt" +) + +type ErrorNoAccess struct { + // Path to directory of repo accessed + Dir string +} + +func (e *ErrorNoAccess) Error() string { + return fmt.Sprintf("Could not access repo at '%s'", e.Dir) +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/events.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/events.go new file mode 100644 index 000000000000..c8b577b1a6cd --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/events.go @@ -0,0 +1,81 @@ +package githttp + +import ( + "fmt" + "net/http" +) + +// An event (triggered on push/pull) +type Event struct { + // One of tag/push/fetch + Type EventType `json:"type"` + + //// + // Set for pushes and pulls + //// + + // SHA of commit + Commit string `json:"commit"` + + // Path to bare repo + Dir string + + //// + // Set for pushes or tagging + //// + Tag string `json:"tag,omitempty"` + Last string `json:"last,omitempty"` + Branch string `json:"branch,omitempty"` + + // Error contains the error that happened (if any) + // during this action/event + Error error + + // Http stuff + Request *http.Request +} + +type EventType int + +// Possible event types +const ( + TAG = iota + 1 + PUSH + FETCH + PUSH_FORCE +) + +func (e EventType) String() string { + switch e { + case TAG: + return "tag" + case PUSH: + return "push" + case PUSH_FORCE: + return "push-force" + case FETCH: + return "fetch" + } + return "unknown" +} + +func (e EventType) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, e)), nil +} + +func (e EventType) UnmarshalJSON(data []byte) error { + str := string(data[:]) + switch str { + case "tag": + e = TAG + case "push": + e = PUSH + case "push-force": + e = PUSH_FORCE + case "fetch": + e = FETCH + default: + return fmt.Errorf("'%s' is not a known git event type") + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/git_reader.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/git_reader.go new file mode 100644 index 000000000000..acafb451b6ff --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/git_reader.go @@ -0,0 +1,49 @@ +package githttp + +import ( + "errors" + "io" + "regexp" +) + +// GitReader scans for errors in the output of a git command +type GitReader struct { + // Underlaying reader (to relay calls to) + io.ReadCloser + + // Error + GitError error +} + +// Regex to detect errors +var ( + gitErrorRegex = regexp.MustCompile("error: (.*)") +) + +// Implement the io.Reader interface +func (g *GitReader) Read(p []byte) (n int, err error) { + // Relay call + n, err = g.ReadCloser.Read(p) + + // Scan for errors + g.scan(p) + + return n, err +} + +func (g *GitReader) scan(data []byte) { + // Already got an error + // the main error will be the first error line + if g.GitError != nil { + return + } + + matches := gitErrorRegex.FindSubmatch(data) + + // Skip, no matches found + if matches == nil { + return + } + + g.GitError = errors.New(string(matches[1][:])) +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/githttp.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/githttp.go new file mode 100644 index 000000000000..bea031ad2cec --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/githttp.go @@ -0,0 +1,291 @@ +package githttp + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "strings" +) + +type GitHttp struct { + // Root directory to serve repos from + ProjectRoot string + + // Path to git binary + GitBinPath string + + // Access rules + UploadPack bool + ReceivePack bool + + // Event handling functions + EventHandler func(ev Event) +} + +// Implement the http.Handler interface +func (g *GitHttp) ServeHTTP(w http.ResponseWriter, r *http.Request) { + g.requestHandler(w, r) + return +} + +// Shorthand constructor for most common scenario +func New(root string) *GitHttp { + return &GitHttp{ + ProjectRoot: root, + GitBinPath: "/usr/bin/git", + UploadPack: true, + ReceivePack: true, + } +} + +// Build root directory if doesn't exist +func (g *GitHttp) Init() (*GitHttp, error) { + if err := os.MkdirAll(g.ProjectRoot, os.ModePerm); err != nil { + return nil, err + } + return g, nil +} + +// Publish event if EventHandler is set +func (g *GitHttp) event(e Event) { + if g.EventHandler != nil { + g.EventHandler(e) + } else { + fmt.Printf("EVENT: %q\n", e) + } +} + +// Actual command handling functions + +func (g *GitHttp) serviceRpc(hr HandlerReq) error { + w, r, rpc, dir := hr.w, hr.r, hr.Rpc, hr.Dir + + access, err := g.hasAccess(r, dir, rpc, true) + if err != nil { + return err + } + + if access == false { + return &ErrorNoAccess{hr.Dir} + } + + // Reader that decompresses if necessary + reader, err := requestReader(r) + if err != nil { + return err + } + + // Reader that scans for events + rpcReader := &RpcReader{ + ReadCloser: reader, + Rpc: rpc, + } + + // Set content type + w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-result", rpc)) + + args := []string{rpc, "--stateless-rpc", "."} + cmd := exec.Command(g.GitBinPath, args...) + cmd.Dir = dir + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + defer stdin.Close() + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + defer stdout.Close() + + err = cmd.Start() + if err != nil { + return err + } + + // Scan's git command's output for errors + gitReader := &GitReader{ + ReadCloser: stdout, + } + + // Copy input to git binary + io.Copy(stdin, rpcReader) + + // Write git binary's output to http response + io.Copy(w, gitReader) + + // Wait till command has completed + mainError := cmd.Wait() + + if mainError == nil { + mainError = gitReader.GitError + } + + // Fire events + for _, e := range rpcReader.Events { + // Set directory to current repo + e.Dir = dir + e.Request = hr.r + e.Error = mainError + + // Fire event + g.event(e) + } + + // May be nil if all is good + return mainError +} + +func (g *GitHttp) getInfoRefs(hr HandlerReq) error { + w, r, dir := hr.w, hr.r, hr.Dir + service_name := getServiceType(r) + access, err := g.hasAccess(r, dir, service_name, false) + if err != nil { + return err + } + + if !access { + g.updateServerInfo(dir) + hdrNocache(w) + return sendFile("text/plain; charset=utf-8", hr) + } + + args := []string{service_name, "--stateless-rpc", "--advertise-refs", "."} + refs, err := g.gitCommand(dir, args...) + if err != nil { + return err + } + + hdrNocache(w) + w.Header().Set("Content-Type", fmt.Sprintf("application/x-git-%s-advertisement", service_name)) + w.WriteHeader(http.StatusOK) + w.Write(packetWrite("# service=git-" + service_name + "\n")) + w.Write(packetFlush()) + w.Write(refs) + + return nil +} + +func (g *GitHttp) getInfoPacks(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("text/plain; charset=utf-8", hr) +} + +func (g *GitHttp) getLooseObject(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-loose-object", hr) +} + +func (g *GitHttp) getPackFile(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-packed-objects", hr) +} + +func (g *GitHttp) getIdxFile(hr HandlerReq) error { + hdrCacheForever(hr.w) + return sendFile("application/x-git-packed-objects-toc", hr) +} + +func (g *GitHttp) getTextFile(hr HandlerReq) error { + hdrNocache(hr.w) + return sendFile("text/plain", hr) +} + +// Logic helping functions + +func sendFile(content_type string, hr HandlerReq) error { + w, r := hr.w, hr.r + req_file := path.Join(hr.Dir, hr.File) + + f, err := os.Stat(req_file) + if err != nil { + return err + } + + w.Header().Set("Content-Type", content_type) + w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Size())) + w.Header().Set("Last-Modified", f.ModTime().Format(http.TimeFormat)) + http.ServeFile(w, r, req_file) + + return nil +} + +func (g *GitHttp) getGitDir(file_path string) (string, error) { + root := g.ProjectRoot + + if root == "" { + cwd, err := os.Getwd() + + if err != nil { + return "", err + } + + root = cwd + } + + f := path.Join(root, file_path) + if _, err := os.Stat(f); os.IsNotExist(err) { + return "", err + } + + return f, nil +} + +func (g *GitHttp) hasAccess(r *http.Request, dir string, rpc string, check_content_type bool) (bool, error) { + if check_content_type { + if r.Header.Get("Content-Type") != fmt.Sprintf("application/x-git-%s-request", rpc) { + return false, nil + } + } + + if !(rpc == "upload-pack" || rpc == "receive-pack") { + return false, nil + } + if rpc == "receive-pack" { + return g.ReceivePack, nil + } + if rpc == "upload-pack" { + return g.UploadPack, nil + } + + return g.getConfigSetting(rpc, dir) +} + +func (g *GitHttp) getConfigSetting(service_name string, dir string) (bool, error) { + service_name = strings.Replace(service_name, "-", "", -1) + setting, err := g.getGitConfig("http."+service_name, dir) + if err != nil { + return false, nil + } + + if service_name == "uploadpack" { + return setting != "false", nil + } + + return setting == "true", nil +} + +func (g *GitHttp) getGitConfig(config_name string, dir string) (string, error) { + args := []string{"config", config_name} + out, err := g.gitCommand(dir, args...) + if err != nil { + return "", err + } + return string(out)[0 : len(out)-1], nil +} + +func (g *GitHttp) updateServerInfo(dir string) ([]byte, error) { + args := []string{"update-server-info"} + return g.gitCommand(dir, args...) +} + +func (g *GitHttp) gitCommand(dir string, args ...string) ([]byte, error) { + command := exec.Command(g.GitBinPath, args...) + command.Dir = dir + + return command.Output() +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/routing.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/routing.go new file mode 100644 index 000000000000..2cd393053566 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/routing.go @@ -0,0 +1,117 @@ +package githttp + +import ( + "net/http" + "os" + "regexp" + "strings" +) + +type Service struct { + Method string + Handler func(HandlerReq) error + Rpc string +} + +type HandlerReq struct { + w http.ResponseWriter + r *http.Request + Rpc string + Dir string + File string +} + +// Routing regexes +var ( + _serviceRpcUpload = regexp.MustCompile("(.*?)/git-upload-pack$") + _serviceRpcReceive = regexp.MustCompile("(.*?)/git-receive-pack$") + _getInfoRefs = regexp.MustCompile("(.*?)/info/refs$") + _getHead = regexp.MustCompile("(.*?)/HEAD$") + _getAlternates = regexp.MustCompile("(.*?)/objects/info/alternates$") + _getHttpAlternates = regexp.MustCompile("(.*?)/objects/info/http-alternates$") + _getInfoPacks = regexp.MustCompile("(.*?)/objects/info/packs$") + _getInfoFile = regexp.MustCompile("(.*?)/objects/info/[^/]*$") + _getLooseObject = regexp.MustCompile("(.*?)/objects/[0-9a-f]{2}/[0-9a-f]{38}$") + _getPackFile = regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.pack$") + _getIdxFile = regexp.MustCompile("(.*?)/objects/pack/pack-[0-9a-f]{40}\\.idx$") +) + +func (g *GitHttp) services() map[*regexp.Regexp]Service { + return map[*regexp.Regexp]Service{ + _serviceRpcUpload: Service{"POST", g.serviceRpc, "upload-pack"}, + _serviceRpcReceive: Service{"POST", g.serviceRpc, "receive-pack"}, + _getInfoRefs: Service{"GET", g.getInfoRefs, ""}, + _getHead: Service{"GET", g.getTextFile, ""}, + _getAlternates: Service{"GET", g.getTextFile, ""}, + _getHttpAlternates: Service{"GET", g.getTextFile, ""}, + _getInfoPacks: Service{"GET", g.getInfoPacks, ""}, + _getInfoFile: Service{"GET", g.getTextFile, ""}, + _getLooseObject: Service{"GET", g.getLooseObject, ""}, + _getPackFile: Service{"GET", g.getPackFile, ""}, + _getIdxFile: Service{"GET", g.getIdxFile, ""}, + } +} + +// getService return's the service corresponding to the +// current http.Request's URL +// as well as the name of the repo +func (g *GitHttp) getService(path string) (string, *Service) { + for re, service := range g.services() { + if m := re.FindStringSubmatch(path); m != nil { + return m[1], &service + } + } + + // No match + return "", nil +} + +// Request handling function +func (g *GitHttp) requestHandler(w http.ResponseWriter, r *http.Request) { + // Get service for URL + repo, service := g.getService(r.URL.Path) + + // No url match + if service == nil { + renderNotFound(w) + return + } + + // Bad method + if service.Method != r.Method { + renderMethodNotAllowed(w, r) + return + } + + // Rpc type + rpc := service.Rpc + + // Get specific file + file := strings.Replace(r.URL.Path, repo+"/", "", 1) + + // Resolve directory + dir, err := g.getGitDir(repo) + + // Repo not found on disk + if err != nil { + renderNotFound(w) + return + } + + // Build request info for handler + hr := HandlerReq{w, r, rpc, dir, file} + + // Call handler + if err := service.Handler(hr); err != nil { + if os.IsNotExist(err) { + renderNotFound(w) + return + } + switch err.(type) { + case *ErrorNoAccess: + renderNoAccess(w) + return + } + http.Error(w, err.Error(), 500) + } +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/rpc_reader.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/rpc_reader.go new file mode 100644 index 000000000000..7d0e5ad9314c --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/rpc_reader.go @@ -0,0 +1,117 @@ +package githttp + +import ( + "io" + "regexp" +) + +// RpcReader scans for events in the incoming rpc request data +type RpcReader struct { + // Underlaying reader (to relay calls to) + io.ReadCloser + + // Rpc type (upload-pack or receive-pack) + Rpc string + + // List of events RpcReader has picked up through scanning + // these events do not contain the "Dir" attribute + Events []Event + + // Tracks first event being scanned + first bool +} + +// Regexes to detect types of actions (fetch, push, etc ...) +var ( + receivePackRegex = regexp.MustCompile("([0-9a-fA-F]{40}) ([0-9a-fA-F]{40}) refs\\/(heads|tags)\\/(.*?)( |00|\u0000)|^(0000)$") + uploadPackRegex = regexp.MustCompile("^\\S+ ([0-9a-fA-F]{40})") +) + +// Implement the io.Reader interface +func (r *RpcReader) Read(p []byte) (n int, err error) { + // Relay call + n, err = r.ReadCloser.Read(p) + + // Scan for events + if err != nil { + r.scan(p) + } + + return n, err +} + +func (r *RpcReader) scan(p []byte) { + events := []Event{} + + switch r.Rpc { + case "receive-pack": + events = scanPush(p) + if !r.first && len(events) == 0 { + events = scanPushForce(p) + r.first = true + } + case "upload-pack": + events = scanFetch(p) + } + + // Add new events + if len(events) > 0 { + r.Events = append(r.Events, events...) + } +} + +func scanFetch(data []byte) []Event { + matches := uploadPackRegex.FindAllStringSubmatch(string(data[:]), -1) + + if matches == nil { + return []Event{} + } + + events := []Event{} + for _, m := range matches { + events = append(events, Event{ + Type: FETCH, + Commit: m[1], + }) + } + + return events +} + +func scanPush(data []byte) []Event { + matches := receivePackRegex.FindAllStringSubmatch(string(data[:]), -1) + + if matches == nil { + return []Event{} + } + + events := []Event{} + for _, m := range matches { + e := Event{ + Last: m[1], + Commit: m[2], + } + + // Handle pushes to branches and tags differently + if m[3] == "heads" { + e.Type = PUSH + e.Branch = m[4] + } else { + e.Type = TAG + e.Tag = m[4] + } + + events = append(events, e) + } + + return events +} + +func scanPushForce(data []byte) []Event { + return []Event{ + Event{ + Type: PUSH_FORCE, + Commit: "HEAD", + }, + } +} diff --git a/Godeps/_workspace/src/github.com/AaronO/go-git-http/utils.go b/Godeps/_workspace/src/github.com/AaronO/go-git-http/utils.go new file mode 100644 index 000000000000..545e9e37d307 --- /dev/null +++ b/Godeps/_workspace/src/github.com/AaronO/go-git-http/utils.go @@ -0,0 +1,93 @@ +package githttp + +import ( + "compress/flate" + "compress/gzip" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// requestReader returns an io.ReadCloser +// that will decode data if needed, depending on the +// "content-encoding" header +func requestReader(req *http.Request) (io.ReadCloser, error) { + switch req.Header.Get("content-encoding") { + case "gzip": + return gzip.NewReader(req.Body) + case "deflate": + return flate.NewReader(req.Body), nil + } + + // If no encoding, use raw body + return req.Body, nil +} + +// HTTP parsing utility functions + +func getServiceType(r *http.Request) string { + service_type := r.FormValue("service") + + if s := strings.HasPrefix(service_type, "git-"); !s { + return "" + } + + return strings.Replace(service_type, "git-", "", 1) +} + +// HTTP error response handling functions + +func renderMethodNotAllowed(w http.ResponseWriter, r *http.Request) { + if r.Proto == "HTTP/1.1" { + w.WriteHeader(http.StatusMethodNotAllowed) + w.Write([]byte("Method Not Allowed")) + } else { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + } +} + +func renderNotFound(w http.ResponseWriter) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Not Found")) +} + +func renderNoAccess(w http.ResponseWriter) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) +} + +// Packet-line handling function + +func packetFlush() []byte { + return []byte("0000") +} + +func packetWrite(str string) []byte { + s := strconv.FormatInt(int64(len(str)+4), 16) + + if len(s)%4 != 0 { + s = strings.Repeat("0", 4-len(s)%4) + s + } + + return []byte(s + str) +} + +// Header writing functions + +func hdrNocache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} + +func hdrCacheForever(w http.ResponseWriter) { + now := time.Now().Unix() + expires := now + 31536000 + w.Header().Set("Date", fmt.Sprintf("%d", now)) + w.Header().Set("Expires", fmt.Sprintf("%d", expires)) + w.Header().Set("Cache-Control", "public, max-age=31536000") +} diff --git a/examples/gitserver/Dockerfile b/examples/gitserver/Dockerfile new file mode 100644 index 000000000000..e971f2c00082 --- /dev/null +++ b/examples/gitserver/Dockerfile @@ -0,0 +1,13 @@ +# +# This is an example Git server for OpenShift Origin. +# +# The standard name for this image is openshift/origin-gitserver +# +FROM openshift/origin + +ADD hooks/ /var/lib/git-hooks/ +RUN ln -s /usr/bin/openshift /usr/bin/openshift-gitserver && \ + mkdir -p /var/lib/git +VOLUME /var/lib/git + +ENTRYPOINT ["/usr/bin/openshift-gitserver"] diff --git a/examples/gitserver/README.md b/examples/gitserver/README.md new file mode 100644 index 000000000000..8fbfdfbbfda9 --- /dev/null +++ b/examples/gitserver/README.md @@ -0,0 +1,16 @@ +Configurable Git Server +======================= + +This example provides automatic mirroring of Git repositories, intended +for use within a container or Kubernetes pod. It can clone repositories +from remote systems on startup as well as remotely register hooks. It +can automatically initialize and receive Git directories on push. + +In the more advanced modes, it can integrate with an OpenShift server to +automatically perform actions when new repositories are created, like +reading the build configs in the current namespace and performing +automatic mirroring of their input, and creating new build-configs when +content is pushed. + +The Dockerfile built by this example is published as +openshift/origin-gitserver \ No newline at end of file diff --git a/examples/gitserver/hooks/detect-language b/examples/gitserver/hooks/detect-language new file mode 100755 index 000000000000..cbbb72851d5f --- /dev/null +++ b/examples/gitserver/hooks/detect-language @@ -0,0 +1,48 @@ +#!/bin/bash +# +# detect-language returns a string indicating the image repository to use as a base +# image for this source repository. Return "docker.io/*/*" for Docker images, a two +# segment entry for a local image repository, or a single segment name to search +# in the current namespace. Set a tag to qualify the version - e.g. "ruby:1.9.3", +# "nodejs:0.10". +# + +set -o errexit +set -o nounset +set -o pipefail + +function has { + [[ -n $(git ls-tree --full-name --name-only HEAD ${@:1}) ]] +} +function key { + git config --local --get "${1}" +} + +prefix=${PREFIX:-openshift/} + +if has Gemfile; then + echo "${prefix}ruby" + exit 0 +fi + +if has requirements.txt; then + echo "${prefix}python" + exit 0 +fi + +if has package.json app.json; then + echo "${prefix}nodejs" + exit 0 +fi + +if has '*.go'; then + echo "${prefix}golang" + exit 0 +fi + +if has index.php; then + echo "${prefix}php" + exit 0 +fi + +exit 1 \ No newline at end of file diff --git a/examples/gitserver/hooks/post-receive b/examples/gitserver/hooks/post-receive new file mode 100755 index 000000000000..f52d500e6186 --- /dev/null +++ b/examples/gitserver/hooks/post-receive @@ -0,0 +1,54 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +function key { + git config --local --get "${1}" +} +function addkey { + git config --local --add "${1}" "${2}" +} + +function detect { + if detected=$(key openshift.io.detect); then + exit 0 + fi + if ! url=$(key gitserver.self.url); then + echo "detect: no self url set" + exit 0 + fi + + # TODO: make it easier to find the build config name created + # by osc new-app + name=$(basename "${url}") + name="${name%.*}" + + if ! lang=$($(dirname $0)/detect-language); then + exit 0 + fi + echo "detect: found language ${lang} for ${name}" + + if ! osc=$(which osc); then + echo "detect: osc is not installed" + addkey openshift.io.detect 1 + exit 0 + fi + osc new-app "${lang}~${url}" + if webhook=$(osc start-build --list-webhooks="generic" "${name}" | head -n 1); then + addkey openshift.io.webhook "${webhook}" + fi + addkey openshift.io.detect 1 +} + +cat > /tmp/postreceived + +detect + +if webhook=$(key openshift.io.webhook); then + # TODO: print output from the server about the hook status + osc start-build --from-webhook="${webhook}" + # TODO: follow logs + echo "build: started" +fi diff --git a/examples/gitserver/main.go b/examples/gitserver/main.go new file mode 100644 index 000000000000..3dbe254578ab --- /dev/null +++ b/examples/gitserver/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "log" + "os" + + "github.com/openshift/origin/pkg/gitserver" +) + +func main() { + if len(os.Args) != 1 { + fmt.Printf(`git-server - Expose Git repositories to the network + +%[1]s`, gitserver.EnvironmentHelp) + os.Exit(0) + } + config, err := gitserver.NewEnviromentConfig() + if err != nil { + log.Fatal(err) + } + log.Fatal(gitserver.Start(config)) +} diff --git a/examples/gitserver/template.yaml b/examples/gitserver/template.yaml new file mode 100644 index 000000000000..3667a13f8e89 --- /dev/null +++ b/examples/gitserver/template.yaml @@ -0,0 +1,74 @@ +apiVersion: v1beta3 +kind: List +items: +- apiVersion: v1beta3 + kind: DeploymentConfig + metadata: + name: gitserver + spec: + triggers: + - type: ConfigChange + replicas: 1 + selector: + run-container: gitserver + template: + metadata: + labels: + run-container: gitserver + spec: + containers: + - name: gitserver + image: openshift/origin-gitserver + ports: + - containerPort: 8080 + env: + - name: PUBLIC_URL + value: "http://gitserver.myproject.local:8080" # TODO: this needs to be resolved from env + - name: GIT_HOME + value: /var/lib/git + - name: HOOK_PATH + value: /var/lib/git-hooks + - name: ALLOW_GIT_PUSH + value: "yes" + - name: ALLOW_GIT_HOOKS + value: "yes" + - name: ALLOW_LAZY_CREATE + value: "yes" + - name: AUTOLINK_CONFIG + value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret + - name: AUTOLINK_NAMESPACE + #value: # TODO: use env generation + - name: AUTOLINK_HOOK + value: + - name: REQUIRE_GIT_AUTH + #value: user:password # if set, authentication is required to push to this server + #- name: GIT_INITIAL_CLONE_1 + # value: + - name: OPENSHIFTCONFIG + value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret + volumeMounts: + - name: config + mountPath: /var/lib/gitsecrets/ + readOnly: true + volumes: + - name: config + secret: + secretName: gitserver-config +- apiVersion: v1beta3 + kind: Secret + metadata: + name: gitserver-config + spec: + data: + # Needs to be populated + admin.kubeconfig: +- apiVersion: v1beta3 + kind: Service + metadata: + name: gitserver + spec: + selector: + run-container: gitserver + ports: + - port: 8080 + targetPort: 8080 diff --git a/hack/build-images.sh b/hack/build-images.sh index 78e9387945b7..f3c6a13d6533 100755 --- a/hack/build-images.sh +++ b/hack/build-images.sh @@ -59,6 +59,7 @@ image openshift/origin-docker-registry images/dockerregistry # images that depend on openshift/origin image openshift/origin-deployer images/deployer image openshift/origin-docker-builder images/builder/docker/docker-builder +image openshift/origin-gitserver examples/gitserver image openshift/origin-sti-builder images/builder/docker/sti-builder # extra images (not part of infrastructure) image openshift/hello-openshift examples/hello-openshift diff --git a/hack/common.sh b/hack/common.sh index ac9ddd8a22ca..a5463ce6cfad 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -55,6 +55,7 @@ readonly OPENSHIFT_BINARY_SYMLINKS=( openshift-deploy openshift-sti-build openshift-docker-build + openshift-gitserver osc osadm ) diff --git a/images/dockerregistry/Dockerfile b/images/dockerregistry/Dockerfile index c7c88acf23f0..9bc08a1e34be 100644 --- a/images/dockerregistry/Dockerfile +++ b/images/dockerregistry/Dockerfile @@ -1,3 +1,9 @@ +# +# This is the integrated OpenShift Origin Docker registry. It is configured to +# publish metadata to OpenShift to provide automatic management of images on push. +# +# The standard name for this image is openshift/origin-docker-registry +# FROM openshift/origin-base ADD config.yml /config.yml diff --git a/pkg/build/api/types.go b/pkg/build/api/types.go index f4f8fa60952e..4a76229407e6 100644 --- a/pkg/build/api/types.go +++ b/pkg/build/api/types.go @@ -349,6 +349,17 @@ type GenericWebHookEvent struct { type GitInfo struct { GitBuildSource `json:",inline"` GitSourceRevision `json:",inline"` + + // Refs is a list of GitRefs for the provided repo - generally sent + // when used from a post-receive hook. This field is optional and is + // used when sending multiple refs + Refs []GitRefInfo `json:"refs,omitempty"` +} + +// GitRefInfo is a single ref +type GitRefInfo struct { + GitBuildSource `json:",inline"` + GitSourceRevision `json:",inline"` } // BuildLog is the (unused) resource associated with the build log redirector diff --git a/pkg/build/webhook/generic/fixtures/post-receive-git.json b/pkg/build/webhook/generic/fixtures/post-receive-git.json new file mode 100644 index 000000000000..d44b46cde30c --- /dev/null +++ b/pkg/build/webhook/generic/fixtures/post-receive-git.json @@ -0,0 +1,22 @@ +{ + "type": "Git", + "git": { + "author": {}, + "committer": {}, + "refs": [ + { + "ref": "refs/heads/master", + "commit": "2602ace61490de0513dfbd7c7de949356cf9bd17", + "author": { + "name": "Martin Nagy", + "email": "nagy.martin@gmail.com" + }, + "committer": { + "name": "Martin Nagy", + "email": "nagy.martin@gmail.com" + }, + "message": "Merge pull request #31 from mnagy/prepare_for_new_mysql_image\n\nPrepare for new openshift/mysql-55-centos7 image" + } + ] + } +} \ No newline at end of file diff --git a/pkg/build/webhook/generic/generic.go b/pkg/build/webhook/generic/generic.go index 04b483c172bc..4da70daa94c3 100644 --- a/pkg/build/webhook/generic/generic.go +++ b/pkg/build/webhook/generic/generic.go @@ -5,7 +5,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" "github.com/golang/glog" @@ -35,6 +34,13 @@ func (p *WebHookPlugin) Extract(buildCfg *api.BuildConfig, secret, path string, if err = verifyRequest(req); err != nil { return } + + git := buildCfg.Parameters.Source.Git + if git == nil { + glog.V(4).Infof("No source defined for build config, but triggering anyway: %s", buildCfg.Name) + return nil, true, nil + } + if req.Body != nil { body, err := ioutil.ReadAll(req.Body) if err != nil { @@ -51,18 +57,27 @@ func (p *WebHookPlugin) Extract(buildCfg *api.BuildConfig, secret, path string, if data.Git == nil { return nil, true, nil } - if !webhook.GitRefMatches(data.Git.Ref, buildCfg.Parameters.Source.Git.Ref) { - glog.V(2).Infof("Skipping build for '%s'. Branch reference from '%s' does not match configuration", buildCfg, data) + + if data.Git.Refs != nil { + for _, ref := range data.Git.Refs { + if webhook.GitRefMatches(ref.Ref, git.Ref) { + revision = &api.SourceRevision{ + Type: api.BuildSourceGit, + Git: &ref.GitSourceRevision, + } + return revision, true, nil + } + } + glog.V(2).Infof("Skipping build for %q. None of the supplied refs matched %q", buildCfg, git.Ref) + return nil, false, nil + } + if !webhook.GitRefMatches(data.Git.Ref, git.Ref) { + glog.V(2).Infof("Skipping build for %q. Branch reference from %q does not match configuration", buildCfg.Name, data.Git.Ref) return nil, false, nil } revision = &api.SourceRevision{ Type: api.BuildSourceGit, - Git: &api.GitSourceRevision{ - Commit: data.Git.Commit, - Message: data.Git.Message, - Author: data.Git.Author, - Committer: data.Git.Committer, - }, + Git: &data.Git.GitSourceRevision, } } return revision, true, nil @@ -72,13 +87,8 @@ func verifyRequest(req *http.Request) error { if method := req.Method; method != "POST" { return fmt.Errorf("Unsupported HTTP method %s", method) } - if userAgent := req.Header.Get("User-Agent"); len(strings.TrimSpace(userAgent)) == 0 { - return fmt.Errorf("User-Agent must be populated with a non-empty value") - } - if contentLength := req.Header.Get("Content-Length"); strings.TrimSpace(contentLength) != "" { - if contentType := req.Header.Get("Content-Type"); contentType != "application/json" { - return fmt.Errorf("Unsupported Content-Type %s", contentType) - } + if contentType := req.Header.Get("Content-Type"); contentType != "application/json" { + return fmt.Errorf("Unsupported Content-Type %s", contentType) } return nil } diff --git a/pkg/build/webhook/generic/generic_test.go b/pkg/build/webhook/generic/generic_test.go index a471ee944558..cb2a3ee7143c 100644 --- a/pkg/build/webhook/generic/generic_test.go +++ b/pkg/build/webhook/generic/generic_test.go @@ -38,6 +38,18 @@ func GivenRequestWithPayload(t *testing.T) *http.Request { return req } +func GivenRequestWithRefsPayload(t *testing.T) *http.Request { + data, err := ioutil.ReadFile("fixtures/post-receive-git.json") + if err != nil { + t.Errorf("Error reading setup data: %v", err) + return nil + } + req, _ := http.NewRequest("POST", "http://someurl.com", bytes.NewReader(data)) + req.Header.Add("User-Agent", "Some User Agent") + req.Header.Add("Content-Type", "application/json") + return req +} + func TestVerifyRequestForMethod(t *testing.T) { req := GivenRequest("GET") err := verifyRequest(req) @@ -47,22 +59,25 @@ func TestVerifyRequestForMethod(t *testing.T) { } func TestVerifyRequestForUserAgent(t *testing.T) { - req := GivenRequest("POST") + req := &http.Request{ + Header: http.Header{"Content-Type": {"application/json"}}, + Method: "POST", + } err := verifyRequest(req) - if err == nil || !strings.Contains(err.Error(), "User-Agent") { - t.Errorf("Exp. User-Agent to be required %v", err) + if err != nil { + t.Errorf("unexpected error %v", err) } req.Header.Add("User-Agent", "") err = verifyRequest(req) - if err == nil || !strings.Contains(err.Error(), "User-Agent") { - t.Errorf("Exp. User-Agent to not empty %v", err) + if err != nil { + t.Errorf("unexpected error %v", err) } req.Header.Set("User-Agent", "foobar") err = verifyRequest(req) - if err != nil && strings.Contains(err.Error(), "User-Agent") { - t.Errorf("Exp. non-empty User-Agent to be valid %v", err) + if err != nil { + t.Errorf("unexpected error %v", err) } } @@ -73,8 +88,8 @@ func TestVerifyRequestForContentType(t *testing.T) { Method: "POST", } err := verifyRequest(req) - if err != nil && strings.Contains(err.Error(), "Content-Type") { - t.Errorf("Exp. a valid request if no payload is posted") + if err != nil && !strings.Contains(err.Error(), "Content-Type") { + t.Errorf("Exp. a content type error") } req.Header.Add("Content-Length", "1") @@ -203,6 +218,77 @@ func TestExtractWithGitPayload(t *testing.T) { } } +func TestExtractWithGitRefsPayload(t *testing.T) { + req := GivenRequestWithRefsPayload(t) + buildConfig := &api.BuildConfig{ + Triggers: []api.BuildTriggerPolicy{ + { + Type: api.GenericWebHookBuildTriggerType, + GenericWebHook: &api.WebHookTrigger{ + Secret: "secret100", + }, + }, + }, + Parameters: api.BuildParameters{ + Source: api.BuildSource{ + Type: api.BuildSourceGit, + Git: &api.GitBuildSource{ + Ref: "master", + }, + }, + Strategy: mockBuildStrategy, + }, + } + plugin := New() + + revision, proceed, err := plugin.Extract(buildConfig, "secret100", "", req) + + if err != nil { + t.Errorf("Expected to be able to trigger a build without a payload error: %v", err) + } + if !proceed { + t.Error("Expected 'proceed' return value to be 'true'") + } + if revision == nil { + t.Error("Expected the 'revision' return value to not be nil") + } +} + +func TestExtractWithUnmatchedGitRefsPayload(t *testing.T) { + req := GivenRequestWithRefsPayload(t) + buildConfig := &api.BuildConfig{ + Triggers: []api.BuildTriggerPolicy{ + { + Type: api.GenericWebHookBuildTriggerType, + GenericWebHook: &api.WebHookTrigger{ + Secret: "secret100", + }, + }, + }, + Parameters: api.BuildParameters{ + Source: api.BuildSource{ + Type: api.BuildSourceGit, + Git: &api.GitBuildSource{ + Ref: "other", + }, + }, + Strategy: mockBuildStrategy, + }, + } + plugin := New() + revision, proceed, err := plugin.Extract(buildConfig, "secret100", "", req) + + if err != nil { + t.Errorf("Expected to be able to trigger a build without a payload error: %v", err) + } + if proceed { + t.Error("Expected 'proceed' return value to be 'false'") + } + if revision != nil { + t.Error("Expected the 'revision' return value to be nil") + } +} + type errJSON struct{} func (*errJSON) Read(p []byte) (n int, err error) { diff --git a/pkg/cmd/cli/cmd/startbuild.go b/pkg/cmd/cli/cmd/startbuild.go index 636810aeb93c..64f3c2d651d8 100644 --- a/pkg/cmd/cli/cmd/startbuild.go +++ b/pkg/cmd/cli/cmd/startbuild.go @@ -2,11 +2,16 @@ package cmd import ( "bytes" + "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "net/url" + "os" + "strings" + "github.com/golang/glog" "github.com/spf13/cobra" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -17,6 +22,7 @@ import ( buildapi "github.com/openshift/origin/pkg/build/api" osclient "github.com/openshift/origin/pkg/client" "github.com/openshift/origin/pkg/cmd/util/clientcmd" + "github.com/openshift/origin/pkg/generate/git" ) const ( @@ -56,6 +62,7 @@ func NewCmdStartBuild(fullName string, f *clientcmd.Factory, out io.Writer) *cob cmd.Flags().Var(&webhooks, "list-webhooks", "List the webhooks for the specified build config or build; accepts 'all', 'generic', or 'github'") cmd.Flags().String("from-webhook", "", "Specify a webhook URL for an existing build config to trigger") cmd.Flags().String("git-post-receive", "", "The contents of the post-receive hook to trigger a build") + cmd.Flags().String("git-repository", "", "The path to the git repository for post-receive; defaults to the current directory") return cmd } @@ -70,7 +77,10 @@ func RunStartBuild(f *clientcmd.Factory, out io.Writer, cmd *cobra.Command, args if len(args) > 0 || len(buildName) > 0 { return cmdutil.UsageError(cmd, "The '--from-webhook' flag is incompatible with arguments or '--from-build'") } - return RunStartBuildWebHook(f, out, webhook, cmdutil.GetFlagString(cmd, "git-post-receive")) + path := cmdutil.GetFlagString(cmd, "git-repository") + postReceivePath := cmdutil.GetFlagString(cmd, "git-post-receive") + repo := git.NewRepository() + return RunStartBuildWebHook(f, out, webhook, path, postReceivePath, repo) case len(args) != 1 && len(buildName) == 0: return cmdutil.UsageError(cmd, "Must pass a name of a build config or specify build name with '--from-build' flag") } @@ -200,35 +210,23 @@ func RunListBuildWebHooks(f *clientcmd.Factory, out, errOut io.Writer, name stri // RunStartBuildWebHook tries to trigger the provided webhook. It will attempt to utilize the current client // configuration if the webhook has the same URL. -func RunStartBuildWebHook(f *clientcmd.Factory, out io.Writer, webhook string, postReceivePath string) error { - // attempt to extract a post receive body - // TODO: implement in follow on - /*refs := []git.ChangedRef{} - switch receive := postReceivePath; { - case receive == "-": - r, err := git.ParsePostReceive(os.Stdin) - if err != nil { - return err - } - refs = r - case len(receive) > 0: - file, err := os.Open(receive) - if err != nil { - return fmt.Errorf("unable to open --git-post-receive argument as a file: %v", err) - } - defer file.Close() - r, err := git.ParsePostReceive(file) - if err != nil { - return err - } - refs = r +func RunStartBuildWebHook(f *clientcmd.Factory, out io.Writer, webhook string, path, postReceivePath string, repo git.Repository) error { + hook, err := url.Parse(webhook) + if err != nil { + return err } - _ = refs*/ - hook, err := url.Parse(webhook) + event, err := hookEventFromPostReceive(repo, path, postReceivePath) if err != nil { return err } + + // TODO: should be a versioned struct + data, err := json.Marshal(event) + if err != nil { + return err + } + httpClient := http.DefaultClient // when using HTTPS, try to reuse the local config transport if possible to get a client cert // TODO: search all configs @@ -244,8 +242,87 @@ func RunStartBuildWebHook(f *clientcmd.Factory, out io.Writer, webhook string, p } } } - if _, err := httpClient.Post(hook.String(), "application/json", bytes.NewBufferString("{}")); err != nil { + glog.V(4).Infof("Triggering hook %s\n%s", hook, string(data)) + resp, err := httpClient.Post(hook.String(), "application/json", bytes.NewBuffer(data)) + if err != nil { return err } + switch { + case resp.StatusCode == 301 || resp.StatusCode == 302: + // TODO: follow redirect and display output + case resp.StatusCode < 200 || resp.StatusCode >= 300: + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("server rejected our request %d\nremote: %s", resp.StatusCode, string(body)) + } return nil } + +// hookEventFromPostReceive creates a GenericWebHookEvent from the provided git repository and +// post receive input. If no inputs are available will return an empty event. +func hookEventFromPostReceive(repo git.Repository, path, postReceivePath string) (*buildapi.GenericWebHookEvent, error) { + // TODO: support other types of refs + event := &buildapi.GenericWebHookEvent{ + Type: buildapi.BuildSourceGit, + Git: &buildapi.GitInfo{}, + } + + // attempt to extract a post receive body + refs := []git.ChangedRef{} + switch receive := postReceivePath; { + case receive == "-": + r, err := git.ParsePostReceive(os.Stdin) + if err != nil { + return nil, err + } + refs = r + case len(receive) > 0: + file, err := os.Open(receive) + if err != nil { + return nil, fmt.Errorf("unable to open --git-post-receive argument as a file: %v", err) + } + defer file.Close() + r, err := git.ParsePostReceive(file) + if err != nil { + return nil, err + } + refs = r + } + for _, ref := range refs { + if len(ref.New) == 0 || ref.New == ref.Old { + continue + } + info, err := gitRefInfo(repo, path, ref.New) + if err != nil { + glog.V(4).Infof("Could not retrieve info for %s:%s: %v", ref.Ref, ref.New, err) + } + info.Ref = ref.Ref + info.Commit = ref.New + event.Git.Refs = append(event.Git.Refs, info) + } + return event, nil +} + +// gitRefInfo extracts a buildapi.GitRefInfo from the specified repository or returns +// an error. +func gitRefInfo(repo git.Repository, dir, ref string) (buildapi.GitRefInfo, error) { + info := buildapi.GitRefInfo{} + if repo == nil { + return info, nil + } + out, err := repo.ShowFormat(dir, ref, "%an%n%ae%n%cn%n%ce%n%B") + if err != nil { + return info, err + } + lines := strings.SplitN(out, "\n", 5) + if len(lines) != 5 { + full := make([]string, 5) + copy(full, lines) + lines = full + } + info.Author.Name = lines[0] + info.Author.Email = lines[1] + info.Committer.Name = lines[2] + info.Committer.Email = lines[3] + info.Message = lines[4] + return info, nil +} diff --git a/pkg/cmd/cli/cmd/startbuild_test.go b/pkg/cmd/cli/cmd/startbuild_test.go index a241f2228852..6dbcdbb43f9a 100644 --- a/pkg/cmd/cli/cmd/startbuild_test.go +++ b/pkg/cmd/cli/cmd/startbuild_test.go @@ -2,15 +2,20 @@ package cmd import ( "bytes" + "encoding/json" "errors" + "fmt" + "io/ioutil" "net/http" "net/http/httptest" + "os" "strings" "testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" + buildapi "github.com/openshift/origin/pkg/build/api" "github.com/openshift/origin/pkg/cmd/util/clientcmd" ) @@ -47,12 +52,12 @@ func TestStartBuildWebHook(t *testing.T) { cfg := &FakeClientConfig{} f := clientcmd.NewFactory(cfg) buf := &bytes.Buffer{} - if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", ""); err != nil { + if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", "", "", nil); err != nil { t.Fatalf("unable to start hook: %v", err) } <-invoked - if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", "unknownpath"); err != nil { + if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", "", "unknownpath", nil); err == nil { t.Fatalf("unexpected non-error: %v", err) } } @@ -71,7 +76,45 @@ func TestStartBuildWebHookHTTPS(t *testing.T) { } f := clientcmd.NewFactory(cfg) buf := &bytes.Buffer{} - if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", ""); err == nil || !strings.Contains(err.Error(), "certificate signed by unknown authority") { + if err := RunStartBuildWebHook(f, buf, server.URL+"/webhook", "", "", nil); err == nil || !strings.Contains(err.Error(), "certificate signed by unknown authority") { t.Fatalf("unexpected non-error: %v", err) } } + +func TestStartBuildHookPostReceive(t *testing.T) { + invoked := make(chan *buildapi.GenericWebHookEvent, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + event := buildapi.GenericWebHookEvent{} + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&event); err != nil { + t.Errorf("unmarshal failed: %v", err) + } + invoked <- &event + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + f, _ := ioutil.TempFile("", "test") + defer os.Remove(f.Name()) + fmt.Fprintf(f, `0000 2384 refs/heads/master +2548 2548 refs/heads/stage`) + f.Close() + + testErr := errors.New("not enabled") + cfg := &FakeClientConfig{ + Err: testErr, + } + factory := clientcmd.NewFactory(cfg) + buf := &bytes.Buffer{} + if err := RunStartBuildWebHook(factory, buf, server.URL+"/webhook", "", f.Name(), nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + event := <-invoked + if event == nil || event.Git == nil || len(event.Git.Refs) != 1 { + t.Fatalf("unexpected event: %#v", event) + } + if event.Git.Refs[0].Commit != "2384" { + t.Fatalf("unexpected ref: %#v", event.Git.Refs[0]) + } +} diff --git a/pkg/cmd/infra/gitserver/gitserver.go b/pkg/cmd/infra/gitserver/gitserver.go new file mode 100644 index 000000000000..6f4ee0c93253 --- /dev/null +++ b/pkg/cmd/infra/gitserver/gitserver.go @@ -0,0 +1,61 @@ +package gitserver + +import ( + "fmt" + "log" + "net/url" + + cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util" + "github.com/spf13/cobra" + + "github.com/openshift/origin/pkg/gitserver" + "github.com/openshift/origin/pkg/gitserver/autobuild" +) + +const longCommandDesc = ` +Start a Git server + +This command launches a Git HTTP/HTTPS server that supports push and pull, mirroring, +and automatic creation of OpenShift applications on push. + +%[1]s +` + +// NewCommandGitServer launches a Git server +func NewCommandGitServer(name string) *cobra.Command { + cmd := &cobra.Command{ + Use: name, + Short: "Start a Git server", + Long: fmt.Sprintf(longCommandDesc, gitserver.EnvironmentHelp), + Run: func(c *cobra.Command, args []string) { + err := RunGitServer() + cmdutil.CheckErr(err) + }, + } + + return cmd +} + +func RunGitServer() error { + config, err := gitserver.NewEnviromentConfig() + if err != nil { + return err + } + link, err := autobuild.NewAutoLinkBuildsFromEnvironment() + switch { + case err == autobuild.ErrNotEnabled: + case err != nil: + log.Fatal(err) + default: + link.LinkFn = func(name string) *url.URL { return gitserver.RepositoryURL(config, name, nil) } + clones, err := link.Link() + if err != nil { + log.Printf("error: %v", err) + break + } + for name, v := range clones { + config.InitialClones[name] = v + } + } + return gitserver.Start(config) +} diff --git a/pkg/cmd/openshift/openshift.go b/pkg/cmd/openshift/openshift.go index 5fa493b8889f..32e364471906 100644 --- a/pkg/cmd/openshift/openshift.go +++ b/pkg/cmd/openshift/openshift.go @@ -20,6 +20,7 @@ import ( "github.com/openshift/origin/pkg/cmd/flagtypes" "github.com/openshift/origin/pkg/cmd/infra/builder" "github.com/openshift/origin/pkg/cmd/infra/deployer" + "github.com/openshift/origin/pkg/cmd/infra/gitserver" "github.com/openshift/origin/pkg/cmd/infra/router" "github.com/openshift/origin/pkg/cmd/server/start" "github.com/openshift/origin/pkg/cmd/templates" @@ -59,6 +60,8 @@ func CommandFor(basename string) *cobra.Command { cmd = builder.NewCommandSTIBuilder(basename) case "openshift-docker-build": cmd = builder.NewCommandDockerBuilder(basename) + case "openshift-gitserver": + cmd = gitserver.NewCommandGitServer(basename) case "osc": cmd = cli.NewCommandCLI(basename, basename) case "osadm": @@ -104,6 +107,7 @@ func NewCommandOpenShift() *cobra.Command { deployer.NewCommandDeployer("deploy"), builder.NewCommandSTIBuilder("sti-build"), builder.NewCommandDockerBuilder("docker-build"), + gitserver.NewCommandGitServer("git-server"), ) root.AddCommand(infra) diff --git a/pkg/generate/app/app.go b/pkg/generate/app/app.go index d63120c6cc7b..65497219fc1c 100644 --- a/pkg/generate/app/app.go +++ b/pkg/generate/app/app.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "net/url" - "path" "reflect" "strconv" "strings" @@ -19,6 +18,7 @@ import ( buildapi "github.com/openshift/origin/pkg/build/api" deployapi "github.com/openshift/origin/pkg/deploy/api" + "github.com/openshift/origin/pkg/generate/git" imageapi "github.com/openshift/origin/pkg/image/api" ) @@ -70,15 +70,10 @@ func (g *Generated) WithType(slicePtr interface{}) bool { func nameFromGitURL(url *url.URL) (string, bool) { // from path - if len(url.Path) > 0 { - base := path.Base(url.Path) - if len(base) > 0 && base != "/" { - if ext := path.Ext(base); ext == ".git" { - base = base[:len(base)-4] - } - return base, true - } + if name, ok := git.NameFromRepositoryURL(url); ok { + return name, true } + // TODO: path is questionable if len(url.Host) > 0 { // from host with port if host, _, err := net.SplitHostPort(url.Host); err == nil { diff --git a/pkg/generate/app/sourcelookup.go b/pkg/generate/app/sourcelookup.go index f699ef66dcdc..a266ecb6e29b 100644 --- a/pkg/generate/app/sourcelookup.go +++ b/pkg/generate/app/sourcelookup.go @@ -5,9 +5,7 @@ import ( "io/ioutil" "net/url" "os" - "path/filepath" "regexp" - "strings" "github.com/openshift/origin/pkg/generate/dockerfile" "github.com/openshift/origin/pkg/generate/git" @@ -44,41 +42,9 @@ type SourceRepository struct { // NewSourceRepository creates a reference to a local or remote source code repository from // a URL or path. func NewSourceRepository(s string) (*SourceRepository, error) { - var location *url.URL - switch { - case strings.HasPrefix(s, "git@"): - base := "git://" + strings.TrimPrefix(s, "git@") - url, err := url.Parse(base) - if err != nil { - return nil, err - } - location = url - - default: - uri, err := url.Parse(s) - if err != nil { - return nil, err - } - - if uri.Scheme == "" { - path := s - ref := "" - segments := strings.SplitN(path, "#", 2) - if len(segments) == 2 { - path, ref = segments[0], segments[1] - } - path, err := filepath.Abs(path) - if err != nil { - return nil, err - } - uri = &url.URL{ - Scheme: "file", - Path: path, - Fragment: ref, - } - } - - location = uri + location, err := git.ParseRepository(s) + if err != nil { + return nil, err } return &SourceRepository{ location: s, diff --git a/pkg/generate/app/test/fakegit.go b/pkg/generate/app/test/fakegit.go index 999b0157a9d1..b144db7a4f94 100644 --- a/pkg/generate/app/test/fakegit.go +++ b/pkg/generate/app/test/fakegit.go @@ -25,7 +25,36 @@ func (g *FakeGit) Clone(dir string, url string) error { return nil } +func (g *FakeGit) CloneBare(dir string, url string) error { + g.CloneCalled = true + return nil +} + +func (g *FakeGit) CloneMirror(source, target string) error { + return nil +} + func (g *FakeGit) Checkout(dir string, ref string) error { g.CheckoutCalled = true return nil } + +func (f *FakeGit) Fetch(source string) error { + return nil +} + +func (f *FakeGit) Init(source string, _ bool) error { + return nil +} + +func (f *FakeGit) AddLocalConfig(source, key, value string) error { + return nil +} + +func (f *FakeGit) AddRemote(source, remote, url string) error { + return nil +} + +func (f *FakeGit) ShowFormat(source, ref, format string) (string, error) { + return "", nil +} diff --git a/pkg/generate/git/git.go b/pkg/generate/git/git.go new file mode 100644 index 000000000000..8cb080f64f2c --- /dev/null +++ b/pkg/generate/git/git.go @@ -0,0 +1,97 @@ +package git + +import ( + "bufio" + "io" + "net/url" + "path" + "path/filepath" + "strings" +) + +// ParseRepository parses a string that may be in the Git format (git@) or URL format +// and extracts the appropriate value. Any fragment on the URL is preserved. +// +// Protocols returned: +// - http, https +// - file +// - git +func ParseRepository(s string) (*url.URL, error) { + switch { + case strings.HasPrefix(s, "git@"): + base := "git://" + strings.TrimPrefix(s, "git@") + url, err := url.Parse(base) + if err != nil { + return nil, err + } + return url, nil + + default: + uri, err := url.Parse(s) + if err != nil { + return nil, err + } + + if uri.Scheme == "" { + path := s + ref := "" + segments := strings.SplitN(path, "#", 2) + if len(segments) == 2 { + path, ref = segments[0], segments[1] + } + path, err := filepath.Abs(path) + if err != nil { + return nil, err + } + uri = &url.URL{ + Scheme: "file", + Path: path, + Fragment: ref, + } + } + + return uri, nil + } +} + +// NameFromRepositoryURL suggests a name for a repository URL based on the last +// segment of the path, or returns false +func NameFromRepositoryURL(url *url.URL) (string, bool) { + // from path + if len(url.Path) > 0 { + base := path.Base(url.Path) + if len(base) > 0 && base != "/" { + if ext := path.Ext(base); ext == ".git" { + base = base[:len(base)-4] + } + return base, true + } + } + return "", false +} + +type ChangedRef struct { + Ref string + Old string + New string +} + +func ParsePostReceive(r io.Reader) ([]ChangedRef, error) { + refs := []ChangedRef{} + scan := bufio.NewScanner(r) + for scan.Scan() { + segments := strings.Split(scan.Text(), " ") + if len(segments) != 3 { + continue + } + refs = append(refs, ChangedRef{ + Ref: segments[2], + Old: segments[0], + New: segments[1], + }) + } + if err := scan.Err(); err != nil && err != io.EOF { + return nil, err + } + return refs, nil +} diff --git a/pkg/generate/git/repository.go b/pkg/generate/git/repository.go index 159631ad859b..e2484776dc96 100644 --- a/pkg/generate/git/repository.go +++ b/pkg/generate/git/repository.go @@ -4,38 +4,71 @@ import ( "bufio" "bytes" "fmt" + "os" "os/exec" + "path/filepath" "regexp" "strings" "unicode" ) -// execCmdFunc is a function that executes an external command -type execCmdFunc func(dir, name string, args ...string) (string, string, error) - // Repository represents a git source repository type Repository interface { GetRootDir(dir string) (string, error) GetOriginURL(dir string) (string, bool, error) GetRef(dir string) string Clone(dir string, url string) error + CloneBare(dir string, url string) error + CloneMirror(dir string, url string) error + Fetch(dir string) error Checkout(dir string, ref string) error + Init(dir string, bare bool) error + AddRemote(dir string, name, url string) error + AddLocalConfig(dir, name, value string) error + ShowFormat(dir, commit, format string) (string, error) } +// execGitFunc is a function that executes a Git command +type execGitFunc func(dir string, args ...string) (string, string, error) + type repository struct { - exec execCmdFunc + git execGitFunc } // NewRepository creates a new Repository for the given directory func NewRepository() Repository { return &repository{ - exec: execCmd, + git: func(dir string, args ...string) (string, string, error) { + return command("git", dir, args...) + }, + } +} + +// NewRepositoryForBinary returns a Repository using the specified +// git executable. +func NewRepositoryForBinary(gitBinaryPath string) Repository { + return &repository{ + git: func(dir string, args ...string) (string, string, error) { + return command(gitBinaryPath, dir, args...) + }, } } +// IsRoot returns true if location is the root of a bare git repository +func IsBareRoot(path string) (bool, error) { + _, err := os.Stat(filepath.Join(path, "HEAD")) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + // GetRootDir obtains the directory root for a Git repository func (r *repository) GetRootDir(location string) (string, error) { - dir, _, err := r.exec(location, "git", "rev-parse", "--git-dir") + dir, _, err := r.git(location, "rev-parse", "--git-dir") if err != nil { return "", err } @@ -61,7 +94,7 @@ var ( // GetOriginURL returns the origin branch URL for the git repository func (r *repository) GetOriginURL(location string) (string, bool, error) { - text, _, err := r.exec(location, "git", "config", "--get-regexp", "^remote\\..*\\.url$") + text, _, err := r.git(location, "config", "--get-regexp", "^remote\\..*\\.url$") if err != nil { return "", false, err } @@ -87,28 +120,71 @@ func (r *repository) GetOriginURL(location string) (string, bool, error) { // GetRef retrieves the current branch reference for the git repository func (r *repository) GetRef(location string) string { - branch, _, err := r.exec(location, "git", "symbolic-ref", "-q", "--short", "HEAD") + branch, _, err := r.git(location, "symbolic-ref", "-q", "--short", "HEAD") if err != nil { branch = "" } return branch } +// AddRemote adds a new remote to the repository. +func (r *repository) AddRemote(location, name, url string) error { + _, _, err := r.git(location, "remote", "add", name, url) + return err +} + +// AddLocalConfig adds a value to the current repository +func (r *repository) AddLocalConfig(location, name, value string) error { + _, _, err := r.git(location, "config", "--local", "--add", name, value) + return err +} + // Clone clones a remote git repository to a local directory func (r *repository) Clone(location string, url string) error { - _, _, err := r.exec("", "git", "clone", "--recursive", url, location) + _, _, err := r.git("", "clone", "--recursive", url, location) + return err +} + +// CloneMirror clones a remote git repository to a local directory as a mirror +func (r *repository) CloneMirror(location string, url string) error { + _, _, err := r.git("", "clone", "--mirror", url, location) + return err +} + +// CloneBare clones a remote git repository to a local directory +func (r *repository) CloneBare(location string, url string) error { + _, _, err := r.git("", "clone", "--bare", url, location) + return err +} + +// Fetch updates the provided git repository +func (r *repository) Fetch(location string) error { + _, _, err := r.git(location, "fetch", "--all") return err } // Checkout switches to the given ref for the git repository func (r *repository) Checkout(location string, ref string) error { - _, _, err := r.exec(location, "git", "checkout", ref) + _, _, err := r.git(location, "checkout", ref) return err } -// execCmd executes an external command in the given directory. +// ShowFormat formats the ref with the given git show format string +func (r *repository) ShowFormat(location, ref, format string) (string, error) { + out, _, err := r.git(location, "show", ref, fmt.Sprintf("--format=%s", format)) + return out, err +} + +// Init initializes a new git repository in the provided location +func (r *repository) Init(location string, bare bool) error { + _, _, err := r.git("", "init", "--bare", location) + return err +} + +// command executes an external command in the given directory. // The command's standard out and error are trimmed and returned as strings -func execCmd(dir, name string, args ...string) (stdout, stderr string, err error) { +// It may return the type *GitError if the command itself fails. +func command(name, dir string, args ...string) (stdout, stderr string, err error) { cmdOut := &bytes.Buffer{} cmdErr := &bytes.Buffer{} @@ -120,5 +196,26 @@ func execCmd(dir, name string, args ...string) (stdout, stderr string, err error err = cmd.Run() stdout = strings.TrimFunc(cmdOut.String(), unicode.IsSpace) stderr = strings.TrimFunc(cmdErr.String(), unicode.IsSpace) + if exitErr, ok := err.(*exec.ExitError); ok { + err = &GitError{ + Err: exitErr, + Stdout: stdout, + Stderr: stderr, + } + } return } + +// GitError is returned when the underlying Git command returns a non-zero exit code. +type GitError struct { + Err error + Stdout string + Stderr string +} + +func (e *GitError) Error() string { + if len(e.Stderr) > 0 { + return e.Stderr + } + return e.Err.Error() +} diff --git a/pkg/generate/git/repository_test.go b/pkg/generate/git/repository_test.go index 15b131d41bd0..c53125644f10 100644 --- a/pkg/generate/git/repository_test.go +++ b/pkg/generate/git/repository_test.go @@ -16,7 +16,7 @@ func TestGetRootDir(t *testing.T) { {"", true, ""}, // When blank is returned, this is not a git repository } for _, test := range tests { - r := &repository{exec: makeExecFunc(test.stdout, nil)} + r := &repository{git: makeExecFunc(test.stdout, nil)} result, err := r.GetRootDir(curDir) if !test.err && err != nil { t.Errorf("Unexpected error: %v", err) @@ -32,7 +32,7 @@ func TestGetRootDir(t *testing.T) { func TestGetOriginURL(t *testing.T) { url := "remote.origin.url https://test.com/a/repository/url" - r := &repository{exec: makeExecFunc(url, nil)} + r := &repository{git: makeExecFunc(url, nil)} result, ok, err := r.GetOriginURL("/test/dir") if err != nil { t.Errorf("Unexpected error: %v", err) @@ -47,7 +47,7 @@ func TestGetOriginURL(t *testing.T) { func TestGetAlterativeOriginURL(t *testing.T) { url := "remote.foo.url https://test.com/a/repository/url\nremote.upstream.url https://test.com/b/repository/url" - r := &repository{exec: makeExecFunc(url, nil)} + r := &repository{git: makeExecFunc(url, nil)} result, ok, err := r.GetOriginURL("/test/dir") if err != nil { t.Errorf("Unexpected error: %v", err) @@ -62,7 +62,7 @@ func TestGetAlterativeOriginURL(t *testing.T) { func TestGetMissingOriginURL(t *testing.T) { url := "remote.foo.url https://test.com/a/repository/url\nremote.bar.url https://test.com/b/repository/url" - r := &repository{exec: makeExecFunc(url, nil)} + r := &repository{git: makeExecFunc(url, nil)} result, ok, err := r.GetOriginURL("/test/dir") if err != nil { t.Errorf("Unexpected error: %v", err) @@ -77,7 +77,7 @@ func TestGetMissingOriginURL(t *testing.T) { func TestGetRef(t *testing.T) { ref := "branch1" - r := &repository{exec: makeExecFunc(ref, nil)} + r := &repository{git: makeExecFunc(ref, nil)} result := r.GetRef("/test/dir") if result != ref { t.Errorf("Unexpected result: %s. Expected: %s", result, ref) @@ -85,7 +85,7 @@ func TestGetRef(t *testing.T) { } func TestClone(t *testing.T) { - r := &repository{exec: makeExecFunc("", nil)} + r := &repository{git: makeExecFunc("", nil)} err := r.Clone("/test/dir", "https://test/url/to/repository") if err != nil { t.Errorf("Unexpected error: %v", err) @@ -93,15 +93,15 @@ func TestClone(t *testing.T) { } func TestCheckout(t *testing.T) { - r := &repository{exec: makeExecFunc("", nil)} + r := &repository{git: makeExecFunc("", nil)} err := r.Checkout("/test/dir", "branch2") if err != nil { t.Errorf("Unexpected error: %v", err) } } -func makeExecFunc(output string, err error) execCmdFunc { - return func(dir, name string, args ...string) (out string, errout string, resultErr error) { +func makeExecFunc(output string, err error) execGitFunc { + return func(dir string, args ...string) (out string, errout string, resultErr error) { out = output resultErr = err return diff --git a/pkg/gitserver/autobuild/autobuild.go b/pkg/gitserver/autobuild/autobuild.go new file mode 100644 index 000000000000..5dba15de8094 --- /dev/null +++ b/pkg/gitserver/autobuild/autobuild.go @@ -0,0 +1,194 @@ +package autobuild + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + kclientcmd "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" + "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors" + + buildapi "github.com/openshift/origin/pkg/build/api" + "github.com/openshift/origin/pkg/client" + "github.com/openshift/origin/pkg/generate/git" + "github.com/openshift/origin/pkg/gitserver" +) + +type AutoLinkBuilds struct { + Namespaces []string + Builders []kapi.ObjectReference + Client client.BuildConfigsNamespacer + + CurrentNamespace string + + PostReceiveHook string + + LinkFn func(name string) *url.URL +} + +var ErrNotEnabled = fmt.Errorf("not enabled") + +func NewAutoLinkBuildsFromEnvironment() (*AutoLinkBuilds, error) { + config := &AutoLinkBuilds{} + + file := os.Getenv("AUTOLINK_CONFIG") + if len(file) == 0 { + return nil, ErrNotEnabled + } + clientConfig, namespace, err := clientFromConfig(file) + if err != nil { + return nil, err + } + client, err := client.New(clientConfig) + if err != nil { + return nil, err + } + config.Client = client + + if value := os.Getenv("AUTOLINK_NAMESPACE"); len(value) > 0 { + namespace = value + } + if len(namespace) == 0 { + return nil, ErrNotEnabled + } + + if value := os.Getenv("AUTOLINK_HOOK"); len(value) > 0 { + abs, err := filepath.Abs(value) + if err != nil { + return nil, err + } + if _, err := os.Stat(abs); err != nil { + return nil, err + } + config.PostReceiveHook = abs + } + + config.Namespaces = []string{namespace} + config.CurrentNamespace = namespace + return config, nil +} + +func clientFromConfig(path string) (*kclient.Config, string, error) { + rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: path} + credentials, err := rules.Load() + if err != nil { + return nil, "", fmt.Errorf("the provided credentials %q could not be loaded: %v", path, err) + } + cfg := kclientcmd.NewDefaultClientConfig(*credentials, &kclientcmd.ConfigOverrides{}) + config, err := cfg.ClientConfig() + if err != nil { + return nil, "", fmt.Errorf("the provided credentials %q could not be used: %v", path, err) + } + namespace, _ := cfg.Namespace() + return config, namespace, nil +} + +func (a *AutoLinkBuilds) Link() (map[string]gitserver.Clone, error) { + errs := []error{} + builders := []*buildapi.BuildConfig{} + for _, namespace := range a.Namespaces { + list, err := a.Client.BuildConfigs(namespace).List(labels.Everything(), fields.Everything()) + if err != nil { + errs = append(errs, err) + continue + } + for i := range list.Items { + builders = append(builders, &list.Items[i]) + } + } + for _, b := range a.Builders { + if hasItem(builders, b) { + continue + } + config, err := a.Client.BuildConfigs(b.Namespace).Get(b.Name) + if err != nil { + errs = append(errs, err) + continue + } + builders = append(builders, config) + } + + hooks := make(map[string]string) + if len(a.PostReceiveHook) > 0 { + hooks["post-receive"] = a.PostReceiveHook + } + + clones := make(map[string]gitserver.Clone) + for _, builder := range builders { + source := builder.Parameters.Source.Git + if source == nil { + continue + } + if builder.Annotations == nil { + builder.Annotations = make(map[string]string) + } + + // calculate the origin URL + uri := source.URI + if value, ok := builder.Annotations["git.openshift.io/origin-url"]; ok { + uri = value + } + if len(uri) == 0 { + continue + } + origin, err := git.ParseRepository(uri) + if err != nil { + errs = append(errs, err) + continue + } + + // calculate the local repository name and self URL + name := builder.Name + if a.CurrentNamespace != builder.Namespace { + name = fmt.Sprintf("%s.%s", builder.Namespace, name) + } + name = fmt.Sprintf("%s.git", name) + self := a.LinkFn(name) + if self == nil { + errs = append(errs, fmt.Errorf("no self URL available, can't update %s", name)) + continue + } + + // we can't clone from ourself + if self.Host == origin.Host { + continue + } + + // update the existing builder + changed := false + if builder.Annotations["git.openshift.io/origin-url"] != origin.String() { + builder.Annotations["git.openshift.io/origin-url"] = origin.String() + changed = true + } + if source.URI != self.String() { + source.URI = self.String() + changed = true + } + if changed { + if _, err := a.Client.BuildConfigs(builder.Namespace).Update(builder); err != nil { + errs = append(errs, err) + continue + } + } + + clones[name] = gitserver.Clone{ + URL: *origin, + Hooks: hooks, + } + } + return clones, errors.NewAggregate(errs) +} + +func hasItem(items []*buildapi.BuildConfig, item kapi.ObjectReference) bool { + for _, c := range items { + if c.Namespace == item.Namespace && c.Name == item.Name { + return true + } + } + return false +} diff --git a/pkg/gitserver/gitserver.go b/pkg/gitserver/gitserver.go new file mode 100644 index 000000000000..c66601549fe2 --- /dev/null +++ b/pkg/gitserver/gitserver.go @@ -0,0 +1,298 @@ +// Package gitserver provides a smart Git HTTP server that can also set and +// remove hooks. The server is lightweight (<7M compiled with a ~2M footprint) +// and can mirror remote repositories in a containerized environment. +package gitserver + +import ( + "fmt" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/AaronO/go-git-http" + "github.com/AaronO/go-git-http/auth" + "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz" + "github.com/prometheus/client_golang/prometheus" + + "github.com/openshift/origin/pkg/generate/git" +) + +const ( + initialClonePrefix = "GIT_INITIAL_CLONE_" + EnvironmentHelp = `Supported environment variables: +GIT_HOME + directory containing Git repositories; defaults to current directory +PUBLIC_URL + the url of this server for constructing URLs that point to this repository +GIT_PATH + path to Git binary; defaults to location of 'git' in PATH +HOOK_PATH + path to a directory containing hooks for all repositories; if not set no global hooks will be used +ALLOW_GIT_PUSH + if 'no', pushes will be not be accepted; defaults to true +ALLOW_GIT_HOOKS + if 'no', hooks cannot be read or set; defaults to true +ALLOW_LAZY_CREATE + if 'no', repositories will not automatically be initialized on push; defaults to true +REQUIRE_GIT_AUTH + a user/password combination required to access the repo of the form ":"; defaults to none +GIT_FORCE_CLEAN + if 'yes', any initial repository directories will be deleted prior to start; defaults to no + WARNING: this is destructive and you will lose any data you have already pushed +GIT_INITIAL_CLONE_*=[;] + each environment variable in this pattern will be cloned when the process starts; failures will be logged + must be [A-Z0-9_\-\.], the cloned directory name will be lowercased. If the name is invalid the + process will halt. If the repository already exists on disk, it will be updated from the remote. +` +) + +var ( + invalidCloneNameChars = regexp.MustCompile("[^a-zA-Z0-9_\\-\\.]") + reservedNames = map[string]struct{}{"_": {}} + + eventCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "git_event_count", + Help: "Counter of events broken out for each repository and type", + }, + []string{"repository", "type"}, + ) +) + +func init() { + prometheus.MustRegister(eventCounter) +} + +// Config represents the configuration to use for running the server +type Config struct { + Home string + GitBinary string + URL *url.URL + + AllowHooks bool + AllowPush bool + AllowLazyCreate bool + + HookDirectory string + MaxHookBytes int64 + + Listen string + + AuthenticatorFn func(http http.Handler) http.Handler + + CleanBeforeClone bool + InitialClones map[string]Clone +} + +// Clone is a repository to clone +type Clone struct { + URL url.URL + Hooks map[string]string +} + +// NewDefaultConfig returns a default server config. +func NewDefaultConfig() *Config { + return &Config{ + Home: "", + GitBinary: "git", + Listen: ":8080", + MaxHookBytes: 50 * 1024, + } +} + +// NewEnviromentConfig sets up the initial config from environment variables +func NewEnviromentConfig() (*Config, error) { + config := NewDefaultConfig() + + home := os.Getenv("GIT_HOME") + if len(home) == 0 { + return nil, fmt.Errorf("GIT_HOME is required") + } + abs, err := filepath.Abs(home) + if err != nil { + return nil, fmt.Errorf("Can't make %q absolute: %v", home, err) + } + if stat, err := os.Stat(abs); err != nil || !stat.IsDir() { + return nil, fmt.Errorf("GIT_HOME must be an existing directory: %v", err) + } + config.Home = home + + if publicURL := os.Getenv("PUBLIC_URL"); len(publicURL) > 0 { + valid, err := url.Parse(publicURL) + if err != nil { + return nil, fmt.Errorf("PUBLIC_URL must be a valid URL: %v", err) + } + config.URL = valid + } + + gitpath := os.Getenv("GIT_PATH") + if len(gitpath) == 0 { + path, err := exec.LookPath("git") + if err != nil { + return nil, fmt.Errorf("could not find 'git' in PATH; specify GIT_PATH or set your PATH") + } + gitpath = path + } + config.GitBinary = gitpath + + config.AllowPush = os.Getenv("ALLOW_GIT_PUSH") != "no" + config.AllowHooks = os.Getenv("ALLOW_GIT_HOOKS") != "no" + config.AllowLazyCreate = os.Getenv("ALLOW_LAZY_CREATE") != "no" + + if hookpath := os.Getenv("HOOK_PATH"); len(hookpath) != 0 { + path, err := filepath.Abs(hookpath) + if err != nil { + return nil, fmt.Errorf("HOOK_PATH was set but cannot be made absolute: %v", err) + } + if stat, err := os.Stat(path); err != nil || !stat.IsDir() { + return nil, fmt.Errorf("HOOK_PATH must be an existing directory if set: %v", err) + } + config.HookDirectory = path + } + + if value := os.Getenv("REQUIRE_GIT_AUTH"); len(value) > 0 { + parts := strings.Split(value, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("REQUIRE_GIT_AUTH must be a username and password separated by a ':'") + } + username, password := parts[0], parts[1] + config.AuthenticatorFn = auth.Authenticator(func(info auth.AuthInfo) (bool, error) { + if info.Push && !config.AllowPush { + return false, nil + } + if info.Username != username || info.Password != password { + return false, nil + } + return true, nil + }) + } + + if value := os.Getenv("GIT_LISTEN"); len(value) > 0 { + config.Listen = value + } + + config.CleanBeforeClone = os.Getenv("GIT_FORCE_CLEAN") == "yes" + + clones := make(map[string]Clone) + for _, env := range os.Environ() { + if !strings.HasPrefix(env, initialClonePrefix) { + continue + } + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + continue + } + key, value := parts[0], parts[1] + part := key[len(initialClonePrefix):] + if len(part) == 0 { + continue + } + if len(value) == 0 { + return nil, fmt.Errorf("%s must not have an empty value", key) + } + + defaultName := strings.Replace(strings.ToLower(part), "_", "-", -1) + values := strings.Split(value, ";") + + var uri, name string + switch len(values) { + case 1: + uri, name = values[0], "" + case 2: + uri, name = values[0], values[1] + if len(name) == 0 { + return nil, fmt.Errorf("%s name may not be empty", key) + } + default: + return nil, fmt.Errorf("%s may only have two segments ( or ;)", key) + } + + url, err := git.ParseRepository(uri) + if err != nil { + return nil, fmt.Errorf("%s is not a valid repository URI: %v", key, err) + } + switch url.Scheme { + case "http", "https", "git", "ssh": + default: + return nil, fmt.Errorf("%s %q must be a http, https, git, or ssh URL", key, uri) + } + + if len(name) == 0 { + if n, ok := git.NameFromRepositoryURL(url); ok { + name = n + ".git" + } + } + if len(name) == 0 { + name = defaultName + ".git" + } + + if invalidCloneNameChars.MatchString(name) { + return nil, fmt.Errorf("%s name %q must be only letters, numbers, dashes, or underscores", key, name) + } + if _, ok := reservedNames[name]; ok { + return nil, fmt.Errorf("%s name %q is reserved (%v)", key, name, reservedNames) + } + + clones[name] = Clone{ + URL: *url, + } + } + config.InitialClones = clones + + return config, nil +} + +func handler(config *Config) http.Handler { + git := githttp.New(config.Home) + git.GitBinPath = config.GitBinary + git.UploadPack = config.AllowPush + git.ReceivePack = config.AllowPush + git.EventHandler = func(ev githttp.Event) { + path := ev.Dir + if strings.HasPrefix(path, config.Home+"/") { + path = path[len(config.Home)+1:] + } + eventCounter.WithLabelValues(path, ev.Type.String()).Inc() + } + handler := http.Handler(git) + + if config.AllowLazyCreate { + handler = lazyInitRepositoryHandler(config, handler) + } + + if config.AuthenticatorFn != nil { + handler = config.AuthenticatorFn(handler) + } + return handler +} + +func Start(config *Config) error { + if err := clone(config); err != nil { + return err + } + handler := handler(config) + + ops := http.NewServeMux() + if config.AllowHooks { + ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config)))) + } + /*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL) + io.Copy(os.Stdout, r.Body) + }))*/ + ops.Handle("/metrics", prometheus.UninstrumentedHandler()) + healthz.InstallHandler(ops) + + mux := http.NewServeMux() + mux.Handle("/", prometheus.InstrumentHandler("git", handler)) + mux.Handle("/_/", http.StripPrefix("/_", ops)) + + log.Printf("Serving %s on %s", config.Home, config.Listen) + return http.ListenAndServe(config.Listen, mux) +} diff --git a/pkg/gitserver/hooks.go b/pkg/gitserver/hooks.go new file mode 100644 index 000000000000..a6cae680bf57 --- /dev/null +++ b/pkg/gitserver/hooks.go @@ -0,0 +1,89 @@ +package gitserver + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +func hooksHandler(config *Config) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + segments := strings.Split(r.URL.Path[1:], "/") + for _, s := range segments { + if len(s) == 0 || s == "." || s == ".." { + http.NotFound(w, r) + return + } + } + if !config.AllowPush { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + switch len(segments) { + case 2: + path := filepath.Join(config.Home, segments[0], "hooks", segments[1]) + if segments[0] == "hooks" { + path = filepath.Join(config.HookDirectory, segments[1]) + } + + switch r.Method { + // TODO: support HEAD or prevent GET for security + case "GET": + w.Header().Set("Content-Type", "text/plain") + http.ServeFile(w, r, path) + + case "DELETE": + if err := os.Remove(path); err != nil { + log.Printf("error: attempted to remove %s: %v", path, err) + } + w.WriteHeader(http.StatusNoContent) + + case "PUT": + if stat, err := os.Stat(path); err == nil { + if stat.IsDir() || (stat.Mode()&0111) == 0 { + http.Error(w, fmt.Errorf("only executable hooks can be changed: %v", stat).Error(), http.StatusInternalServerError) + return + } + // unsymlink and overwrite + if (stat.Mode() & os.ModeSymlink) != 0 { + os.Remove(path) + } + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0750) + if err != nil { + http.Error(w, fmt.Errorf("unable to open hook file: %v", err).Error(), http.StatusInternalServerError) + return + } + defer f.Close() + max := config.MaxHookBytes + 1 + body := io.LimitReader(r.Body, max) + buf := make([]byte, max) + n, err := io.ReadFull(body, buf) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + http.Error(w, fmt.Errorf("unable to read hook: %v", err).Error(), http.StatusInternalServerError) + return + } + if int64(n) == max { + http.Error(w, fmt.Errorf("hook was too long, truncated to %d bytes", config.MaxHookBytes).Error(), 422) + } else { + w.WriteHeader(http.StatusOK) + } + if _, err := f.Write(buf[:n]); err != nil { + http.Error(w, fmt.Errorf("unable to write hook: %v", err).Error(), http.StatusInternalServerError) + return + } + + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } + + default: + http.NotFound(w, r) + } + }) +} diff --git a/pkg/gitserver/initializer.go b/pkg/gitserver/initializer.go new file mode 100644 index 000000000000..4f4d1811d887 --- /dev/null +++ b/pkg/gitserver/initializer.go @@ -0,0 +1,213 @@ +package gitserver + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/openshift/origin/pkg/generate/git" +) + +var lazyInitMatch = regexp.MustCompile("^/([^\\/]+?)/info/refs$") + +// lazyInitRepositoryHandler creates a handler that will initialize a Git repository +// if it does not yet exist. +func lazyInitRepositoryHandler(config *Config, handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + handler.ServeHTTP(w, r) + return + } + match := lazyInitMatch.FindStringSubmatch(r.URL.Path) + if match == nil { + handler.ServeHTTP(w, r) + return + } + name := match[1] + if name == "." || name == ".." { + handler.ServeHTTP(w, r) + return + } + path := filepath.Join(config.Home, name) + _, err := os.Stat(path) + if !os.IsNotExist(err) { + handler.ServeHTTP(w, r) + return + } + + self := RepositoryURL(config, name, r) + log.Printf("Lazily initializing bare repository %s", self.String()) + + defaultHooks, err := loadHooks(config.HookDirectory) + if err != nil { + log.Printf("error: unable to load default hooks: %v", err) + http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError) + return + } + + // TODO: capture init hook output for Git + if _, err := newRepository(config, path, defaultHooks, self, nil); err != nil { + log.Printf("error: unable to initialize repo %s: %v", path, err) + http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError) + os.RemoveAll(path) + return + } + eventCounter.WithLabelValues(name, "init").Inc() + + handler.ServeHTTP(w, r) + }) +} + +// RepositoryURL creates the public URL for the named git repo. If both config.URL and +// request are nil, the returned URL will be nil. +func RepositoryURL(config *Config, name string, r *http.Request) *url.URL { + var url url.URL + switch { + case config.URL != nil: + url = *config.URL + case r != nil: + url = *r.URL + url.Host = r.Host + url.Scheme = "http" + default: + return nil + } + url.Path = "/" + name + url.RawQuery = "" + url.Fragment = "" + return &url +} + +func newRepository(config *Config, path string, hooks map[string]string, self *url.URL, origin *url.URL) ([]byte, error) { + var out []byte + repo := git.NewRepositoryForBinary(config.GitBinary) + + if origin != nil { + if err := repo.CloneMirror(path, origin.String()); err != nil { + return out, err + } + } else { + if err := repo.Init(path, true); err != nil { + return out, err + } + } + + if self != nil { + if err := repo.AddLocalConfig(path, "gitserver.self.url", self.String()); err != nil { + return out, err + } + } + + // remove all sample hooks, ignore errors here + if files, err := ioutil.ReadDir(filepath.Join(path, "hooks")); err == nil { + for _, file := range files { + os.Remove(filepath.Join(path, "hooks", file.Name())) + } + } + + for name, hook := range hooks { + dest := filepath.Join(path, "hooks", name) + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return out, err + } + if err := os.Symlink(hook, dest); err != nil { + return out, err + } + } + + if initHook, ok := hooks["init"]; ok { + cmd := exec.Command(initHook) + cmd.Dir = path + result, err := cmd.CombinedOutput() + if err != nil { + return out, fmt.Errorf("init hook failed: %v\n%s", err, string(result)) + } + out = result + } + + return out, nil +} + +// clone clones the provided git repositories +func clone(config *Config) error { + defaultHooks, err := loadHooks(config.HookDirectory) + if err != nil { + return err + } + + errs := []error{} + for name, v := range config.InitialClones { + hooks := mergeHooks(defaultHooks, v.Hooks) + url := v.URL + url.Fragment = "" + path := filepath.Join(config.Home, name) + ok, err := git.IsBareRoot(path) + if err != nil { + errs = append(errs, err) + continue + } + if ok { + if !config.CleanBeforeClone { + continue + } + log.Printf("Removing %s", path) + if err := os.RemoveAll(path); err != nil { + errs = append(errs, err) + continue + } + } + log.Printf("Cloning %s into %s", url.String(), path) + + self := RepositoryURL(config, name, nil) + if _, err := newRepository(config, path, hooks, self, &url); err != nil { + // TODO: tear this directory down + errs = append(errs, err) + continue + } + } + if len(errs) > 0 { + s := []string{} + for _, err := range errs { + s = append(s, err.Error()) + } + return fmt.Errorf("Initial clone failed:\n* %s", strings.Join(s, "\n* ")) + } + return nil +} + +func loadHooks(path string) (map[string]string, error) { + hooks := make(map[string]string) + if len(path) == 0 { + return hooks, nil + } + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + for _, file := range files { + if file.IsDir() || (file.Mode().Perm()&0111) == 0 { + continue + } + hook := filepath.Join(path, file.Name()) + name := filepath.Base(hook) + hooks[name] = hook + } + return hooks, nil +} + +func mergeHooks(hooks ...map[string]string) map[string]string { + hook := make(map[string]string) + for _, m := range hooks { + for k, v := range m { + hook[k] = v + } + } + return hook +}