Skip to content

Commit 72360d0

Browse files
Danieloni1Danieloni1
authored
Merge pull request #3 from globekeeper/daniel/dev-1138-data-integrity
Data Integrity
2 parents 2bb7872 + aa4f010 commit 72360d0

11 files changed

+221
-19
lines changed

‎api/r0/upload_async.go

+45-6
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,81 @@
11
package r0
22

33
import (
4+
"bytes"
45
"errors"
6+
"io"
57
"net/http"
68
"path/filepath"
79

810
"github.com/getsentry/sentry-go"
11+
"github.com/h2non/filetype"
912
"github.com/sirupsen/logrus"
1013
"github.com/t2bot/matrix-media-repo/api/_apimeta"
1114
"github.com/t2bot/matrix-media-repo/api/_responses"
1215
"github.com/t2bot/matrix-media-repo/api/_routers"
1316
"github.com/t2bot/matrix-media-repo/common"
1417
"github.com/t2bot/matrix-media-repo/common/rcontext"
1518
"github.com/t2bot/matrix-media-repo/pipelines/pipeline_upload"
19+
"github.com/t2bot/matrix-media-repo/util"
1620
)
1721

1822
func UploadMediaAsync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
1923
server := _routers.GetParam("server", r)
2024
mediaId := _routers.GetParam("mediaId", r)
2125
filename := filepath.Base(r.URL.Query().Get("filename"))
22-
2326
rctx = rctx.LogWithFields(logrus.Fields{
2427
"mediaId": mediaId,
2528
"server": server,
2629
"filename": filename,
2730
})
28-
29-
if r.Host != server {
31+
// GK CUSTOMIZATION: Sanitize the filename
32+
if len(filename) > rctx.Config.Uploads.MaxFilenameLength {
33+
rctx.Log.Info("Filename too long")
3034
return &_responses.ErrorResponse{
31-
Code: common.ErrCodeNotFound,
32-
Message: "Upload request is for another domain.",
33-
InternalCode: common.ErrCodeForbidden,
35+
Code: common.ErrCodeBadRequest,
36+
Message: "Filename too long.",
37+
InternalCode: common.ErrCodeBadRequest,
3438
}
3539
}
3640

3741
contentType := r.Header.Get("Content-Type")
3842
if contentType == "" {
3943
contentType = "application/octet-stream" // binary
44+
} else {
45+
// GK CUSTOMIZATION: Check if the file type is supported
46+
buf, err := io.ReadAll(r.Body)
47+
r.Body = io.NopCloser(bytes.NewBuffer(buf))
48+
if err != nil {
49+
return &_responses.ErrorResponse{
50+
Code: common.ErrCodeBadRequest,
51+
Message: "Error reading file.",
52+
InternalCode: common.ErrCodeBadRequest,
53+
}
54+
}
55+
kind, err := filetype.Match(buf)
56+
if err != nil {
57+
return &_responses.ErrorResponse{
58+
Code: common.ErrCodeBadRequest,
59+
Message: "Error matching file type.",
60+
InternalCode: common.ErrCodeBadRequest,
61+
}
62+
}
63+
if !util.IsSupportedFileType(kind.Extension, rctx.Config.Uploads.SupportedFileTypes) {
64+
rctx.Log.Info("Unsupported file type: ", kind.Extension)
65+
return &_responses.ErrorResponse{
66+
Code: common.ErrCodeBadRequest,
67+
Message: "Unsupported file type.",
68+
InternalCode: common.ErrCodeBadRequest,
69+
}
70+
}
71+
}
72+
73+
if r.Host != server {
74+
return &_responses.ErrorResponse{
75+
Code: common.ErrCodeNotFound,
76+
Message: "Upload request is for another domain.",
77+
InternalCode: common.ErrCodeForbidden,
78+
}
4079
}
4180

4281
// Early sizing constraints (reject requests which claim to be too large/small)

‎api/r0/upload_sync.go

+40-1
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package r0
22

33
import (
4+
"bytes"
45
"errors"
6+
"io"
57
"net/http"
68
"path/filepath"
79
"strconv"
810

911
"github.com/getsentry/sentry-go"
12+
"github.com/h2non/filetype"
1013
"github.com/sirupsen/logrus"
1114
"github.com/t2bot/matrix-media-repo/api/_apimeta"
1215
"github.com/t2bot/matrix-media-repo/api/_responses"
@@ -23,14 +26,50 @@ type MediaUploadedResponse struct {
2326

2427
func UploadMediaSync(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
2528
filename := filepath.Base(r.URL.Query().Get("filename"))
26-
2729
rctx = rctx.LogWithFields(logrus.Fields{
2830
"filename": filename,
2931
})
32+
// GK CUSTOMIZATION: Sanitize the filename
33+
if len(filename) > rctx.Config.Uploads.MaxFilenameLength {
34+
rctx.Log.Info("Filename too long")
35+
return &_responses.ErrorResponse{
36+
Code: common.ErrCodeBadRequest,
37+
Message: "Filename too long.",
38+
InternalCode: common.ErrCodeBadRequest,
39+
}
40+
}
3041

3142
contentType := r.Header.Get("Content-Type")
3243
if contentType == "" {
3344
contentType = "application/octet-stream" // binary
45+
} else {
46+
// GK CUSTOMIZATION: Check if the file type is supported
47+
buf, err := io.ReadAll(r.Body)
48+
r.Body = io.NopCloser(bytes.NewBuffer(buf))
49+
if err != nil {
50+
return &_responses.ErrorResponse{
51+
Code: common.ErrCodeBadRequest,
52+
Message: "Error reading file.",
53+
InternalCode: common.ErrCodeBadRequest,
54+
}
55+
}
56+
kind, err := filetype.Match(buf)
57+
if err != nil {
58+
return &_responses.ErrorResponse{
59+
Code: common.ErrCodeBadRequest,
60+
Message: "Error matching file type.",
61+
InternalCode: common.ErrCodeBadRequest,
62+
}
63+
}
64+
if !util.IsSupportedFileType(kind.Extension, rctx.Config.Uploads.SupportedFileTypes) {
65+
rctx.Log.Info("Unsupported file type: ", kind.Extension)
66+
return &_responses.ErrorResponse{
67+
Code: common.ErrCodeBadRequest,
68+
Message: "Unsupported file type.",
69+
InternalCode: common.ErrCodeBadRequest,
70+
}
71+
}
72+
//
3473
}
3574

3675
// Early sizing constraints (reject requests which claim to be too large/small)

‎api/routes.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func buildRoutes() http.Handler {
3232
// Standard (spec) features
3333
register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter))
3434
register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter))
35-
downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMedia), "download", counter)
35+
downloadRoute := makeRoute(_routers.RequireAccessToken(r0.DownloadMedia), "download", counter)
3636
register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute)
3737
register([]string{"GET"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute)
3838
register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter))

‎common/config/conf_min_shared.go

+34
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,40 @@ func NewDefaultMinimumRepoConfig() MinimumRepoConfig {
2020
TargetBytesPerPart: 209715200, // 200mb
2121
},
2222
Uploads: UploadsConfig{
23+
MaxFilenameLength: 24,
24+
SupportedFileTypes: []string{
25+
"docx",
26+
"doc",
27+
"xlsx",
28+
"xls",
29+
"odt",
30+
"ods",
31+
"odp",
32+
"csv",
33+
"txt",
34+
"pdf",
35+
"ppr",
36+
"pptx",
37+
"ppt",
38+
"jpg",
39+
"png",
40+
"gif",
41+
"heic",
42+
"bmp",
43+
"webp",
44+
"svg",
45+
"mpeg4",
46+
"mpeg",
47+
"h264",
48+
"webm",
49+
"mkv",
50+
"avi",
51+
"mov",
52+
"mp3",
53+
"wav",
54+
"flac",
55+
"zip",
56+
},
2357
MaxSizeBytes: 104857600, // 100mb
2458
MinSizeBytes: 100,
2559
ReportedMaxSizeBytes: 0,

‎common/config/models_domain.go

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ type QuotasConfig struct {
1919
}
2020

2121
type UploadsConfig struct {
22+
MaxFilenameLength int `yaml:"maxFilenameLength"`
23+
SupportedFileTypes []string `yaml:"supportedFileTypes"`
2224
MaxSizeBytes int64 `yaml:"maxBytes"`
2325
MinSizeBytes int64 `yaml:"minBytes"`
2426
ReportedMaxSizeBytes int64 `yaml:"reportedMaxBytes"`

‎config.sample.yaml

+36
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,42 @@ archiving:
232232

233233
# The file upload settings for the media repository
234234
uploads:
235+
# GK CUSTOMIZATION
236+
maxFilenameLength: 24
237+
supportedFileTypes:
238+
- "docx"
239+
- "doc"
240+
- "xlsx"
241+
- "xls"
242+
- "odt"
243+
- "ods"
244+
- "odp"
245+
- "csv"
246+
- "txt"
247+
- "pdf"
248+
- "ppr"
249+
- "pptx"
250+
- "ppt"
251+
- "jpg"
252+
- "png"
253+
- "gif"
254+
- "heic"
255+
- "bmp"
256+
- "webp"
257+
- "svg"
258+
- "mpeg4"
259+
- "mpeg"
260+
- "h264"
261+
- "webm"
262+
- "mkv"
263+
- "avi"
264+
- "mov"
265+
- "mp3"
266+
- "wav"
267+
- "flac"
268+
- "zip"
269+
##################
270+
235271
# The maximum individual file size a user can upload.
236272
maxBytes: 104857600 # 100MB default, 0 to disable
237273

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ require (
151151
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
152152
github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
153153
github.com/go-sql-driver/mysql v1.5.0 // indirect
154+
github.com/h2non/filetype v1.1.3
154155
github.com/hashicorp/yamux v0.1.1 // indirect
155156
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
156157
github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect

0 commit comments

Comments
 (0)