From fd45e692f67b6d404ae91176941c4aae4a7465c8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 22 Jan 2022 10:03:50 +0000 Subject: [PATCH 1/4] Set the LastModified header for raw files Although the use of LastModified dates for caching of git objects should be discouraged (as it is not native to git - and there are a LOT of ways this could be incorrect) - LastModified dates can be a helpful somewhat more human way of caching for simple cases. This PR adds this header and handles the If-Modified-Since header to the /raw/ routes. Fix #18354 Signed-off-by: Andrew Thornton --- modules/httpcache/httpcache.go | 9 ++++-- routers/web/repo/download.go | 58 ++++++++++++++++++++++++---------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 11b63148d9d7..fb2b63095253 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -37,18 +37,23 @@ func generateETag(fi os.FileInfo) string { // HandleTimeCache handles time-based caching for a HTTP request func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) { + return HandleGenericTimeCache(req, w, fi.ModTime()) +} + +// HandleGenericTimeCache handles time-based caching for a HTTP request +func HandleGenericTimeCache(req *http.Request, w http.ResponseWriter, lastModified time.Time) (handled bool) { AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) ifModifiedSince := req.Header.Get("If-Modified-Since") if ifModifiedSince != "" { t, err := time.Parse(http.TimeFormat, ifModifiedSince) - if err == nil && fi.ModTime().Unix() <= t.Unix() { + if err == nil && lastModified.Unix() <= t.Unix() { w.WriteHeader(http.StatusNotModified) return true } } - w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) + w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) return false } diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 72d34cb93793..33820f8d7c60 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -6,7 +6,10 @@ package repo import ( + "path" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -79,34 +82,57 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { return common.ServeBlob(ctx, blob) } +func getBlobForEntry(ctx *context.Context) *git.Blob { + entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) + if err != nil { + ctx.ServerError("GetBlobByPath", err) + return nil + } + + if entry.IsDir() || entry.IsSubModule() { + ctx.NotFound("GetBlobByPath", nil) + return nil + } + + var c *git.LastCommitCache + if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + } + + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + if err != nil { + ctx.ServerError("GetCommitsInfo", err) + return nil + } + + if len(info) == 1 && httpcache.HandleGenericTimeCache(ctx.Req, ctx.Resp, info[0].Commit.Committer.When) { + // Not Modified + return nil + } + + return entry.Blob() +} + // SingleDownload download a file by repos path func SingleDownload(ctx *context.Context) { - blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath) - if err != nil { - if git.IsErrNotExist(err) { - ctx.NotFound("GetBlobByPath", nil) - } else { - ctx.ServerError("GetBlobByPath", err) - } + blob := getBlobForEntry(ctx) + if blob == nil { return } - if err = common.ServeBlob(ctx, blob); err != nil { + + if err := common.ServeBlob(ctx, blob); err != nil { ctx.ServerError("ServeBlob", err) } } // SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary func SingleDownloadOrLFS(ctx *context.Context) { - blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath) - if err != nil { - if git.IsErrNotExist(err) { - ctx.NotFound("GetBlobByPath", nil) - } else { - ctx.ServerError("GetBlobByPath", err) - } + blob := getBlobForEntry(ctx) + if blob == nil { return } - if err = ServeBlobOrLFS(ctx, blob); err != nil { + + if err := ServeBlobOrLFS(ctx, blob); err != nil { ctx.ServerError("ServeBlobOrLFS", err) } } From 3a9c8c8238fff255df82844ac39a02b8e6f71143 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 23 Jan 2022 19:46:46 +0000 Subject: [PATCH 2/4] as per review Signed-off-by: Andrew Thornton --- routers/web/repo/download.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 33820f8d7c60..264a031b7ab2 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -85,12 +85,12 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { func getBlobForEntry(ctx *context.Context) *git.Blob { entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { - ctx.ServerError("GetBlobByPath", err) + ctx.ServerError("GetTreeEntryByPath", err) return nil } if entry.IsDir() || entry.IsSubModule() { - ctx.NotFound("GetBlobByPath", nil) + ctx.NotFound("getBlobForEntry", nil) return nil } From ff236a5d3a0dab8ffe47ea9811b1d36b085995a7 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 24 Jan 2022 23:37:41 +0000 Subject: [PATCH 3/4] Ensure that If-None-Match overrides If-Modified-Since Signed-off-by: Andrew Thornton --- modules/httpcache/httpcache.go | 30 +++++++++++++++++++++++ routers/api/v1/repo/file.go | 44 +++++++++++++++++++++++++++------- routers/common/repo.go | 5 ++-- routers/web/repo/download.go | 36 +++++++++++++++------------- routers/web/repo/wiki.go | 3 ++- 5 files changed, 90 insertions(+), 28 deletions(-) diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index fb2b63095253..5797e981cf80 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -90,3 +90,33 @@ func checkIfNoneMatchIsValid(req *http.Request, etag string) bool { } return false } + +// HandleGenericETagTimeCache handles ETag-based caching with Last-Modified caching for a HTTP request. +// It returns true if the request was handled. +func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag string, lastModified time.Time) (handled bool) { + if len(etag) > 0 { + w.Header().Set("Etag", etag) + } + if !lastModified.IsZero() { + w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) + } + + if len(etag) > 0 { + if checkIfNoneMatchIsValid(req, etag) { + w.WriteHeader(http.StatusNotModified) + return true + } + } + if !lastModified.IsZero() { + ifModifiedSince := req.Header.Get("If-Modified-Since") + if ifModifiedSince != "" { + t, err := time.Parse(http.TimeFormat, ifModifiedSince) + if err == nil && lastModified.Unix() <= t.Unix() { + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) + return false +} diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index a27e383bc31c..324518bb7f7b 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -9,13 +9,16 @@ import ( "encoding/base64" "fmt" "net/http" + "path" "time" "code.gitea.io/gitea/models" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" @@ -62,6 +65,14 @@ func GetRawFile(ctx *context.APIContext) { return } + blob, lastModified := getBlobForEntry(ctx) + + if err := common.ServeBlob(ctx.Context, blob, lastModified); err != nil { + ctx.Error(http.StatusInternalServerError, "ServeBlob", err) + } +} + +func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, lastModified time.Time) { commit := ctx.Repo.Commit if ref := ctx.FormTrim("ref"); len(ref) > 0 { @@ -77,18 +88,35 @@ func GetRawFile(ctx *context.APIContext) { } } - blob, err := commit.GetBlobByPath(ctx.Repo.TreePath) + entry, err := commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { - if git.IsErrNotExist(err) { - ctx.NotFound() - } else { - ctx.Error(http.StatusInternalServerError, "GetBlobByPath", err) - } + ctx.Error(http.StatusInternalServerError, "GetTreeEntryByPath", err) return } - if err = common.ServeBlob(ctx.Context, blob); err != nil { - ctx.Error(http.StatusInternalServerError, "ServeBlob", err) + + if entry.IsDir() || entry.IsSubModule() { + ctx.NotFound("getBlobForEntry", nil) + return + } + + var c *git.LastCommitCache + if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + } + + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetCommitsInfo", err) + return } + + if len(info) == 1 { + // Not Modified + lastModified = info[0].Commit.Committer.When + } + blob = entry.Blob() + + return } // GetArchive get archive of a repository diff --git a/routers/common/repo.go b/routers/common/repo.go index b0e14b63f542..d037e151f933 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "strings" + "time" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" @@ -22,8 +23,8 @@ import ( ) // ServeBlob download a git.Blob -func ServeBlob(ctx *context.Context, blob *git.Blob) error { - if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) { +func ServeBlob(ctx *context.Context, blob *git.Blob, lastModified time.Time) error { + if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) { return nil } diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 264a031b7ab2..4bc40e87ad28 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -7,6 +7,7 @@ package repo import ( "path" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cache" @@ -21,8 +22,8 @@ import ( ) // ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary -func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { - if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) { +func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified time.Time) error { + if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) { return nil } @@ -48,7 +49,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { log.Error("ServeBlobOrLFS: Close: %v", err) } closed = true - return common.ServeBlob(ctx, blob) + return common.ServeBlob(ctx, blob, lastModified) } if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) { return nil @@ -79,19 +80,19 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { } closed = true - return common.ServeBlob(ctx, blob) + return common.ServeBlob(ctx, blob, lastModified) } -func getBlobForEntry(ctx *context.Context) *git.Blob { +func getBlobForEntry(ctx *context.Context) (blob *git.Blob, lastModified time.Time) { entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { ctx.ServerError("GetTreeEntryByPath", err) - return nil + return } if entry.IsDir() || entry.IsSubModule() { ctx.NotFound("getBlobForEntry", nil) - return nil + return } var c *git.LastCommitCache @@ -102,37 +103,38 @@ func getBlobForEntry(ctx *context.Context) *git.Blob { info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) if err != nil { ctx.ServerError("GetCommitsInfo", err) - return nil + return } - if len(info) == 1 && httpcache.HandleGenericTimeCache(ctx.Req, ctx.Resp, info[0].Commit.Committer.When) { + if len(info) == 1 { // Not Modified - return nil + lastModified = info[0].Commit.Committer.When } + blob = entry.Blob() - return entry.Blob() + return } // SingleDownload download a file by repos path func SingleDownload(ctx *context.Context) { - blob := getBlobForEntry(ctx) + blob, lastModified := getBlobForEntry(ctx) if blob == nil { return } - if err := common.ServeBlob(ctx, blob); err != nil { + if err := common.ServeBlob(ctx, blob, lastModified); err != nil { ctx.ServerError("ServeBlob", err) } } // SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary func SingleDownloadOrLFS(ctx *context.Context) { - blob := getBlobForEntry(ctx) + blob, lastModified := getBlobForEntry(ctx) if blob == nil { return } - if err := ServeBlobOrLFS(ctx, blob); err != nil { + if err := ServeBlobOrLFS(ctx, blob, lastModified); err != nil { ctx.ServerError("ServeBlobOrLFS", err) } } @@ -148,7 +150,7 @@ func DownloadByID(ctx *context.Context) { } return } - if err = common.ServeBlob(ctx, blob); err != nil { + if err = common.ServeBlob(ctx, blob, time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } } @@ -164,7 +166,7 @@ func DownloadByIDOrLFS(ctx *context.Context) { } return } - if err = ServeBlobOrLFS(ctx, blob); err != nil { + if err = ServeBlobOrLFS(ctx, blob, time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } } diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index 633458081f81..a6ef7e382134 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -13,6 +13,7 @@ import ( "net/url" "path/filepath" "strings" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/unit" @@ -612,7 +613,7 @@ func WikiRaw(ctx *context.Context) { } if entry != nil { - if err = common.ServeBlob(ctx, entry.Blob()); err != nil { + if err = common.ServeBlob(ctx, entry.Blob(), time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } return From 2d60ac3a69597709a86cbb787372162dfd709ae3 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 9 May 2022 17:25:37 +0200 Subject: [PATCH 4/4] add 404 (back) --- routers/api/v1/repo/file.go | 9 ++++++++- routers/web/repo/download.go | 6 +++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 5961b1afb916..2a4c4ad9797c 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -66,6 +66,9 @@ func GetRawFile(ctx *context.APIContext) { } blob, lastModified := getBlobForEntry(ctx) + if ctx.Written() { + return + } if err := common.ServeBlob(ctx.Context, blob, lastModified); err != nil { ctx.Error(http.StatusInternalServerError, "ServeBlob", err) @@ -75,7 +78,11 @@ func GetRawFile(ctx *context.APIContext) { func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, lastModified time.Time) { entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { - ctx.Error(http.StatusInternalServerError, "GetTreeEntryByPath", err) + if git.IsErrNotExist(err) { + ctx.NotFound() + } else { + ctx.Error(http.StatusInternalServerError, "GetTreeEntryByPath", err) + } return } diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 4bc40e87ad28..eae61bb8e760 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -86,7 +86,11 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified time.Time func getBlobForEntry(ctx *context.Context) (blob *git.Blob, lastModified time.Time) { entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { - ctx.ServerError("GetTreeEntryByPath", err) + if git.IsErrNotExist(err) { + ctx.NotFound("GetTreeEntryByPath", err) + } else { + ctx.ServerError("GetTreeEntryByPath", err) + } return }