Add experimental support for IPFS downloads

Step towards #115

api/features/feature_routes.go

@@ -7,6 +7,10 @@ import (
 const MSC2448UploadRoute = "/_matrix/media/unstable/xyz.amorgan/upload"
 const MSC2448GetHashRoute = "/_matrix/media/unstable/xyz.amorgan/download/{server:[a-zA-Z0-9.:\\-_]+}/{mediaId:[a-zA-Z0-9.\\-_]+}/blurhash"
 const MSC2448AltRenderRoute = "/_matrix/media/unstable/io.t2bot.msc2448/blurhash/{blurhash:[^/]+}"
+const IPFSDownloadRoute = "/_matrix/media/unstable/io.t2bot.ipfs/download/{server:[a-zA-Z0-9.:\\-_]+}/{ipfsContentId:.+}"
+const IPFSLiveDownloadRouteR0 = "/_matrix/media/r0/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:.+}"
+const IPFSLiveDownloadRouteV1 = "/_matrix/media/v1/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:.+}"
+const IPFSLiveDownloadRouteUnstable = "/_matrix/media/v1/download/{server:[a-zA-Z0-9.:\\-_]+}/ipfs:{ipfsContentId:.+}"
 
 func IsRoute(r *http.Request, route string) bool {
 	uri := r.URL.Path

api/unstable/ipfs_download.go

@@ -0,0 +1,54 @@
+package unstable
+
+import (
+	"bytes"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-http-client"
+	"github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/sirupsen/logrus"
+	"github.com/turt2live/matrix-media-repo/api"
+	"github.com/turt2live/matrix-media-repo/api/r0"
+	"github.com/turt2live/matrix-media-repo/common/rcontext"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+func IPFSDownload(r *http.Request, rctx rcontext.RequestContext, user api.UserInfo) interface{} {
+	params := mux.Vars(r)
+
+	server := params["server"]
+	ipfsContentId := params["ipfsContentId"]
+
+	rctx = rctx.LogWithFields(logrus.Fields{
+		"ipfsContentId": ipfsContentId,
+		"server":        server,
+	})
+
+	ipfs, err := httpapi.NewLocalApi()
+	if err != nil {
+		rctx.Log.Error(err)
+		return api.InternalServerError("Unexpected error connecting to IPFS")
+	}
+
+	ipfsCid, err := cid.Decode(ipfsContentId)
+	if err != nil {
+		rctx.Log.Error(err)
+		return api.InternalServerError("Unexpected error decoding content ID")
+	}
+
+	ipfsPath := path.IpfsPath(ipfsCid)
+	node, err := ipfs.ResolveNode(rctx.Context, ipfsPath)
+	if err != nil {
+		rctx.Log.Error(err)
+		return api.InternalServerError("Unexpected error resolving object from IPFS")
+	}
+
+	return &r0.DownloadMediaResponse{
+		ContentType: "application/octet-stream",
+		Filename:    "ipfs.dat", // TODO: Figure out how to get a name out of this
+		SizeBytes:   int64(len(node.RawData())),
+		Data:        util.BufferToStream(bytes.NewBuffer(node.RawData())), // stream to avoid log spam
+	}
+}

api/webserver/webserver.go

@@ -76,6 +76,7 @@ func Init() *sync.WaitGroup {
 	versionHandler := handler{api.AccessTokenOptionalRoute(custom.GetVersion), "get_version", counter, false}
 	blurhashRenderHandler := handler{api.AccessTokenRequiredRoute(unstable.RenderBlurhash), "render_blurhash", counter, false}
 	blurhashCalcHandler := handler{api.AccessTokenRequiredRoute(unstable.GetBlurhash), "calculate_blurhash", counter, false}
+	ipfsDownloadHandler := handler{api.AccessTokenOptionalRoute(unstable.IPFSDownload), "ipfs_download", counter, false}
 
 	routes := make(map[string]route)
 	// r0 is typically clients and v1 is typically servers. v1 is deprecated.
@@ -146,6 +147,13 @@ func Init() *sync.WaitGroup {
 		routes[features.MSC2448AltRenderRoute] = route{"GET", blurhashRenderHandler}
 	}
 
+	if config.Get().Features.IPFS.Enabled {
+		routes[features.IPFSDownloadRoute] = route{"GET", ipfsDownloadHandler}
+		routes[features.IPFSLiveDownloadRouteR0] = route{"GET", ipfsDownloadHandler}
+		routes[features.IPFSLiveDownloadRouteV1] = route{"GET", ipfsDownloadHandler}
+		routes[features.IPFSLiveDownloadRouteUnstable] = route{"GET", ipfsDownloadHandler}
+	}
+
 	for routePath, route := range routes {
 		logrus.Info("Registering route: " + route.method + " " + routePath)
 		rtr.Handle(routePath, route.handler).Methods(route.method)

common/config/conf_min_shared.go

@@ -50,6 +50,9 @@ func NewDefaultMinimumRepoConfig() MinimumRepoConfig {
 				YComponents:     3,
 				Punch:           1,
 			},
+			IPFS: IPFSConfig{
+				Enabled: false,
+			},
 		},
 	}
 }

common/config/models_domain.go

@@ -76,6 +76,7 @@ type TimeoutsConfig struct {
 
 type FeatureConfig struct {
 	MSC2448Blurhash MSC2448Config `yaml:"MSC2448"`
+	IPFS            IPFSConfig    `yaml:"IPFS"`
 }
 
 type MSC2448Config struct {
@@ -88,3 +89,7 @@ type MSC2448Config struct {
 	YComponents     int  `yaml:"yComponents"`
 	Punch           int  `yaml:"punch"`
 }
+
+type IPFSConfig struct {
+	Enabled bool `yaml:"enabled"`
+}

common/config/watch.go

@@ -99,6 +99,9 @@ func hasWebFeatureChanged(configNew *MainRepoConfig, configNow *MainRepoConfig)
 	if configNew.Features.MSC2448Blurhash.Enabled != configNow.Features.MSC2448Blurhash.Enabled {
 		return true
 	}
+	if configNew.Features.IPFS.Enabled != configNow.Features.IPFS.Enabled {
+		return true
+	}
 
 	return false
 }

config.sample.yaml

@@ -426,3 +426,11 @@ featureSupport:
     # The amount of contrast to apply when converting a blurhash to an image. Lower values
     # make the effect more subtle, larger values make it stronger.
     punch: 1
+
+  # IPFS Support
+  # This is currently beta and might not work at all. It also assumes you have a local IPFS
+  # agent running so it can access the HTTP API. In future these options might be moved and
+  # more configurable.
+  IPFS:
+    # Whether or not IPFS support is enabled for use in the media repo.
+    enabled: false

go.mod

@@ -27,6 +27,8 @@ require (
 	github.com/gorilla/mux v1.7.4
 	github.com/h2non/filetype v1.0.12
 	github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c
+	github.com/ipfs/go-ipfs-http-client v0.0.5
+	github.com/ipfs/interface-go-ipfs-core v0.2.6
 	github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect
 	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect
@@ -35,7 +37,6 @@ require (
 	github.com/lib/pq v1.3.0
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/minio/minio-go v6.0.14+incompatible
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee
@@ -53,6 +54,7 @@ require (
 	github.com/tebeka/strftime v0.1.3 // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	google.golang.org/appengine v1.4.0 // indirect
 	gopkg.in/ini.v1 v1.52.0 // indirect
 	gopkg.in/yaml.v2 v2.2.8
 )