Support freezing media on the unauthenticated endpoints (#600)

* Support freezing media on the unauthenticated endpoints * changelog
t2bot · Jul 29, 2024 · 306a2be · 306a2be
1 parent 3272722
commit 306a2be
Show file tree

Hide file tree

Showing 20 changed files with 248 additions and 41 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+* A new global config option, `repo.freezeUnauthenticatedMedia`, is supported to enact the unauthenticated media freeze early.
+
 ### Changed
 
 * The default leaky bucket capacity has changed from 300mb to 500mb, allowing for more downloads to go through. The drain rate and overflow limit are unchanged (5mb/minute and 100mb respectively).

diff --git a/api/_apimeta/auth.go b/api/_apimeta/auth.go
@@ -20,6 +20,15 @@ type ServerInfo struct {
 	ServerName string
 }
 
+type AuthContext struct {
+	User   UserInfo
+	Server ServerInfo
+}
+
+func (a AuthContext) IsAuthenticated() bool {
+	return a.User.UserId != "" || a.Server.ServerName != ""
+}
+
 func GetRequestUserAdminStatus(r *http.Request, rctx rcontext.RequestContext, user UserInfo) (bool, bool) {
 	isGlobalAdmin := util.IsGlobalAdmin(user.UserId) || user.IsShared
 	isLocalAdmin, err := matrix.IsUserAdmin(rctx, r.Host, user.AccessToken, r.RemoteAddr)

diff --git a/api/r0/download.go b/api/r0/download.go
@@ -18,7 +18,11 @@ import (
 	"github.com/t2bot/matrix-media-repo/common/rcontext"
 )
 
-func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+func DownloadMediaUser(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	return DownloadMedia(r, rctx, _apimeta.AuthContext{User: user})
+}
+
+func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, auth _apimeta.AuthContext) interface{} {
 	server := _routers.GetParam("server", r)
 	mediaId := _routers.GetParam("mediaId", r)
 	filename := _routers.GetParam("filename", r)
@@ -61,28 +65,39 @@ func DownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.
 	}
 
 	rctx = rctx.LogWithFields(logrus.Fields{
-		"mediaId":       mediaId,
-		"server":        server,
-		"filename":      filename,
-		"allowRemote":   downloadRemote,
-		"allowRedirect": canRedirect,
+		"mediaId":        mediaId,
+		"server":         server,
+		"filename":       filename,
+		"allowRemote":    downloadRemote,
+		"allowRedirect":  canRedirect,
+		"authUserId":     auth.User.UserId,
+		"authServerName": auth.Server.ServerName,
 	})
 
-	if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) {
-		rctx.Log.Warn("Request blocked due to domain being ignored.")
-		return _responses.MediaBlocked()
+	if auth.User.UserId != "" {
+		if !util.IsGlobalAdmin(auth.User.UserId) && util.IsHostIgnored(server) {
+			rctx.Log.Warn("Request blocked due to domain being ignored.")
+			return _responses.MediaBlocked()
+		}
 	}
 
 	media, stream, err := pipeline_download.Execute(rctx, server, mediaId, pipeline_download.DownloadOpts{
 		FetchRemoteIfNeeded: downloadRemote,
 		BlockForReadUntil:   blockFor,
 		CanRedirect:         canRedirect,
 		RecordOnly:          recordOnly,
+		AuthProvided:        auth.IsAuthenticated(),
 	})
 	if err != nil {
 		var redirect datastores.RedirectError
 		if errors.Is(err, common.ErrMediaNotFound) {
 			return _responses.NotFoundError()
+		} else if errors.Is(err, common.ErrRestrictedAuth) {
+			return _responses.ErrorResponse{
+				Code:         common.ErrCodeNotFound,
+				Message:      "authentication is required to download this media",
+				InternalCode: common.ErrCodeUnauthorized,
+			}
 		} else if errors.Is(err, common.ErrMediaTooLarge) {
 			return _responses.RequestTooLarge()
 		} else if errors.Is(err, common.ErrRateLimitExceeded) {

diff --git a/api/r0/thumbnail.go b/api/r0/thumbnail.go
@@ -20,7 +20,11 @@ import (
 	"github.com/t2bot/matrix-media-repo/common/rcontext"
 )
 
-func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+func ThumbnailMediaUser(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
+	return ThumbnailMedia(r, rctx, _apimeta.AuthContext{User: user})
+}
+
+func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, auth _apimeta.AuthContext) interface{} {
 	server := _routers.GetParam("server", r)
 	mediaId := _routers.GetParam("mediaId", r)
 	allowRemote := r.URL.Query().Get("allow_remote")
@@ -55,15 +59,19 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
 	}
 
 	rctx = rctx.LogWithFields(logrus.Fields{
-		"mediaId":       mediaId,
-		"server":        server,
-		"allowRemote":   downloadRemote,
-		"allowRedirect": canRedirect,
+		"mediaId":        mediaId,
+		"server":         server,
+		"allowRemote":    downloadRemote,
+		"allowRedirect":  canRedirect,
+		"authUserId":     auth.User.UserId,
+		"authServerName": auth.Server.ServerName,
 	})
 
-	if !util.IsGlobalAdmin(user.UserId) && util.IsHostIgnored(server) {
-		rctx.Log.Warn("Request blocked due to domain being ignored.")
-		return _responses.MediaBlocked()
+	if auth.User.UserId != "" {
+		if !util.IsGlobalAdmin(auth.User.UserId) && util.IsHostIgnored(server) {
+			rctx.Log.Warn("Request blocked due to domain being ignored.")
+			return _responses.MediaBlocked()
+		}
 	}
 
 	widthStr := r.URL.Query().Get("width")
@@ -124,6 +132,7 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
 			BlockForReadUntil:   blockFor,
 			RecordOnly:          false, // overridden
 			CanRedirect:         canRedirect,
+			AuthProvided:        auth.IsAuthenticated(),
 		},
 		Width:    width,
 		Height:   height,
@@ -134,6 +143,12 @@ func ThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta
 		var redirect datastores.RedirectError
 		if errors.Is(err, common.ErrMediaNotFound) {
 			return _responses.NotFoundError()
+		} else if errors.Is(err, common.ErrRestrictedAuth) {
+			return _responses.ErrorResponse{
+				Code:         common.ErrCodeNotFound,
+				Message:      "authentication is required to download this media",
+				InternalCode: common.ErrCodeUnauthorized,
+			}
 		} else if errors.Is(err, common.ErrMediaTooLarge) {
 			return _responses.RequestTooLarge()
 		} else if errors.Is(err, common.ErrRateLimitExceeded) {

diff --git a/api/routes.go b/api/routes.go
@@ -33,10 +33,10 @@ func buildRoutes() http.Handler {
 	// Standard (spec) features
 	register([]string{"PUT"}, PrefixMedia, "upload/:server/:mediaId", mxV3, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaAsync), "upload_async", counter))
 	register([]string{"POST"}, PrefixMedia, "upload", mxSpecV3Transition, router, makeRoute(_routers.RequireAccessToken(r0.UploadMediaSync), "upload", counter))
-	downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMedia), "download", counter)
+	downloadRoute := makeRoute(_routers.OptionalAccessToken(r0.DownloadMediaUser), "download", counter)
 	register([]string{"GET", "HEAD"}, PrefixMedia, "download/:server/:mediaId/:filename", mxSpecV3Transition, router, downloadRoute)
 	register([]string{"GET", "HEAD"}, PrefixMedia, "download/:server/:mediaId", mxSpecV3Transition, router, downloadRoute)
-	register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMedia), "thumbnail", counter))
+	register([]string{"GET"}, PrefixMedia, "thumbnail/:server/:mediaId", mxSpecV3Transition, router, makeRoute(_routers.OptionalAccessToken(r0.ThumbnailMediaUser), "thumbnail", counter))
 	previewUrlRoute := makeRoute(_routers.RequireAccessToken(r0.PreviewUrl), "url_preview", counter)
 	register([]string{"GET"}, PrefixMedia, "preview_url", mxSpecV3TransitionCS, router, previewUrlRoute)
 	register([]string{"GET"}, PrefixMedia, "identicon/*seed", mxR0, router, makeRoute(_routers.OptionalAccessToken(r0.Identicon), "identicon", counter))

diff --git a/api/v1/download.go b/api/v1/download.go
@@ -2,9 +2,10 @@ package v1
 
 import (
 	"bytes"
-	"github.com/t2bot/matrix-media-repo/util/ids"
 	"net/http"
 
+	"github.com/t2bot/matrix-media-repo/util/ids"
+
 	"github.com/t2bot/matrix-media-repo/api/_apimeta"
 	"github.com/t2bot/matrix-media-repo/api/_responses"
 	"github.com/t2bot/matrix-media-repo/api/_routers"
@@ -16,7 +17,7 @@ import (
 func ClientDownloadMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
 	r.URL.Query().Set("allow_remote", "true")
 	r.URL.Query().Set("allow_redirect", "true")
-	return r0.DownloadMedia(r, rctx, user)
+	return r0.DownloadMedia(r, rctx, _apimeta.AuthContext{User: user})
 }
 
 func FederationDownloadMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
@@ -26,7 +27,7 @@ func FederationDownloadMedia(r *http.Request, rctx rcontext.RequestContext, serv
 	r.URL.RawQuery = query.Encode()
 	r = _routers.ForceSetParam("server", r.Host, r)
 
-	res := r0.DownloadMedia(r, rctx, _apimeta.UserInfo{})
+	res := r0.DownloadMedia(r, rctx, _apimeta.AuthContext{Server: server})
 	boundary, err := ids.NewUniqueId()
 	if err != nil {
 		rctx.Log.Error("Error generating boundary on response: ", err)

diff --git a/api/v1/thumbnail.go b/api/v1/thumbnail.go
@@ -2,9 +2,10 @@ package v1
 
 import (
 	"bytes"
-	"github.com/t2bot/matrix-media-repo/util/ids"
 	"net/http"
 
+	"github.com/t2bot/matrix-media-repo/util/ids"
+
 	"github.com/t2bot/matrix-media-repo/api/_apimeta"
 	"github.com/t2bot/matrix-media-repo/api/_responses"
 	"github.com/t2bot/matrix-media-repo/api/_routers"
@@ -16,7 +17,7 @@ import (
 func ClientThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, user _apimeta.UserInfo) interface{} {
 	r.URL.Query().Set("allow_remote", "true")
 	r.URL.Query().Set("allow_redirect", "true")
-	return r0.ThumbnailMedia(r, rctx, user)
+	return r0.ThumbnailMedia(r, rctx, _apimeta.AuthContext{User: user})
 }
 
 func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, server _apimeta.ServerInfo) interface{} {
@@ -26,7 +27,7 @@ func FederationThumbnailMedia(r *http.Request, rctx rcontext.RequestContext, ser
 	r.URL.RawQuery = query.Encode()
 	r = _routers.ForceSetParam("server", r.Host, r)
 
-	res := r0.ThumbnailMedia(r, rctx, _apimeta.UserInfo{})
+	res := r0.ThumbnailMedia(r, rctx, _apimeta.AuthContext{Server: server})
 	boundary, err := ids.NewUniqueId()
 	if err != nil {
 		rctx.Log.Error("Error generating boundary on response: ", err)

diff --git a/archival/entity_export.go b/archival/entity_export.go
@@ -44,6 +44,7 @@ func ExportEntityData(ctx rcontext.RequestContext, exportId string, entityId str
 			FetchRemoteIfNeeded: false,
 			BlockForReadUntil:   10 * time.Minute,
 			RecordOnly:          false,
+			AuthProvided:        true, // it's for an export, so assume authentication
 		})
 		if errors.Is(err, common.ErrMediaQuarantined) {
 			ctx.Log.Warnf("%s is quarantined and will not be included in the export", mxc)

diff --git a/common/config/conf_main.go b/common/config/conf_main.go
@@ -24,14 +24,15 @@ func NewDefaultMainConfig() MainRepoConfig {
 	return MainRepoConfig{
 		MinimumRepoConfig: NewDefaultMinimumRepoConfig(),
 		General: GeneralConfig{
-			BindAddress:      "127.0.0.1",
-			Port:             8000,
-			LogDirectory:     "logs",
-			LogColors:        false,
-			JsonLogs:         false,
-			LogLevel:         "info",
-			TrustAnyForward:  false,
-			UseForwardedHost: true,
+			BindAddress:                "127.0.0.1",
+			Port:                       8000,
+			LogDirectory:               "logs",
+			LogColors:                  false,
+			JsonLogs:                   false,
+			LogLevel:                   "info",
+			TrustAnyForward:            false,
+			UseForwardedHost:           true,
+			FreezeUnauthenticatedMedia: false,
 		},
 		Database: DatabaseConfig{
 			Postgres: "postgres://your_username:your_password@localhost/database_name?sslmode=disable",

diff --git a/common/config/models_main.go b/common/config/models_main.go
@@ -1,14 +1,15 @@
 package config
 
 type GeneralConfig struct {
-	BindAddress      string `yaml:"bindAddress"`
-	Port             int    `yaml:"port"`
-	LogDirectory     string `yaml:"logDirectory"`
-	LogColors        bool   `yaml:"logColors"`
-	JsonLogs         bool   `yaml:"jsonLogs"`
-	LogLevel         string `yaml:"logLevel"`
-	TrustAnyForward  bool   `yaml:"trustAnyForwardedAddress"`
-	UseForwardedHost bool   `yaml:"useForwardedHost"`
+	BindAddress                string `yaml:"bindAddress"`
+	Port                       int    `yaml:"port"`
+	LogDirectory               string `yaml:"logDirectory"`
+	LogColors                  bool   `yaml:"logColors"`
+	JsonLogs                   bool   `yaml:"jsonLogs"`
+	LogLevel                   string `yaml:"logLevel"`
+	TrustAnyForward            bool   `yaml:"trustAnyForwardedAddress"`
+	UseForwardedHost           bool   `yaml:"useForwardedHost"`
+	FreezeUnauthenticatedMedia bool   `yaml:"freezeUnauthenticatedMedia"`
 }
 
 type HomeserverConfig struct {

diff --git a/common/errors.go b/common/errors.go
@@ -17,3 +17,4 @@ var ErrAlreadyUploaded = errors.New("already uploaded")
 var ErrMediaNotYetUploaded = errors.New("media not yet uploaded")
 var ErrMediaDimensionsTooSmall = errors.New("media is too small dimensionally")
 var ErrRateLimitExceeded = errors.New("rate limit exceeded")
+var ErrRestrictedAuth = errors.New("authentication is required to download this media")
diff --git a/config.sample.yaml b/config.sample.yaml
@@ -34,6 +34,17 @@ repo:
   # See https://github.com/t2bot/matrix-media-repo/issues/202 for more information.
   useForwardedHost: true
 
+  # If true, media uploaded or cached from that point forwards will require authentication in order to
+  # be accessed. Media uploaded or cached prior will remain accessible on the unauthenticated endpoints.
+  # If set to false after being set to true, media uploaded or cached while the flag was true will still
+  # only be accessible over authenticated endpoints, though future media will be accessible on both
+  # authenticated and unauthenticated media.
+  #
+  # This flag currently defaults to false. A future release, likely in August 2024, will remove this flag
+  # and have the same effect as it being true (always on). This flag is primarily intended for servers to
+  # opt-in to the behaviour early.
+  freezeUnauthenticatedMedia: false
+
 # Options for dealing with federation
 federation:
   # On a per-host basis, the number of consecutive failures in calling the host before the

diff --git a/database/db.go b/database/db.go
@@ -28,6 +28,7 @@ type Database struct {
 	Tasks           *tasksTableStatements
 	Exports         *exportsTableStatements
 	ExportParts     *exportPartsTableStatements
+	RestrictedMedia *restrictedMediaTableStatements
 }
 
 var instance *Database
@@ -126,6 +127,9 @@ func openDatabase(connectionString string, maxConns int, maxIdleConns int) error
 	if d.ExportParts, err = prepareExportPartsTables(d.conn); err != nil {
 		return errors.New("failed to create export parts table accessor: " + err.Error())
 	}
+	if d.RestrictedMedia, err = prepareRestrictedMediaTables(d.conn); err != nil {
+		return errors.New("failed to create restricted media table accessor: " + err.Error())
+	}
 
 	instance = d
 	return nil