diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index adc49e3a9f3..a11eb0351e4 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released [Commits](https://github.com/scalableminds/webknossos/compare/23.02.0...HEAD) ### Added +- Remote datasets can now also be streamed from Google Cloud Storage URIs (`gs://`). [#6775](https://github.com/scalableminds/webknossos/pull/6775) ### Changed - Limit paid team sharing features to respective organization plans. [#6767](https://github.com/scalableminds/webknossos/pull/6776) diff --git a/app/controllers/CredentialController.scala b/app/controllers/CredentialController.scala index af3a56143d5..7089ee19b5d 100644 --- a/app/controllers/CredentialController.scala +++ b/app/controllers/CredentialController.scala @@ -2,28 +2,39 @@ package controllers import com.mohiva.play.silhouette.api.Silhouette import com.scalableminds.util.tools.FoxImplicits -import com.scalableminds.webknossos.datastore.storage.{HttpBasicAuthCredential, S3AccessKeyCredential} +import com.scalableminds.webknossos.datastore.storage.{ + GoogleServiceAccountCredential, + HttpBasicAuthCredential, + S3AccessKeyCredential +} import models.binary.credential.CredentialDAO import oxalis.security.WkEnv -import play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{JsValue, Json, OFormat} import play.api.mvc.{Action, PlayBodyParsers} import utils.ObjectId import javax.inject.Inject import scala.concurrent.ExecutionContext -case class HttpBasicAuthCredentialParameters(name: String, username: String, password: String, domain: Option[String]) +case class HttpBasicAuthCredentialParameters(name: String, username: String, password: String) object HttpBasicAuthCredentialParameters { implicit val jsonFormat: OFormat[HttpBasicAuthCredentialParameters] = Json.format[HttpBasicAuthCredentialParameters] } -case class S3AccessKeyCredentialParameters(name: String, keyId: String, key: String, bucket: Option[String]) +case class S3AccessKeyCredentialParameters(name: String, accessKeyId: String, secretAccessKey: String) object S3AccessKeyCredentialParameters { implicit val jsonFormat: OFormat[S3AccessKeyCredentialParameters] = Json.format[S3AccessKeyCredentialParameters] } +case class GoogleServiceAccountCredentialParameters(name: String, secretJson: JsValue) + +object GoogleServiceAccountCredentialParameters { + implicit val jsonFormat: OFormat[GoogleServiceAccountCredentialParameters] = + Json.format[GoogleServiceAccountCredentialParameters] +} + class CredentialController @Inject()(credentialDAO: CredentialDAO, sil: Silhouette[WkEnv])( implicit ec: ExecutionContext, val bodyParsers: PlayBodyParsers) @@ -54,12 +65,27 @@ class CredentialController @Inject()(credentialDAO: CredentialDAO, sil: Silhouet _ <- credentialDAO.insertOne( _id, S3AccessKeyCredential(request.body.name, - request.body.keyId, - request.body.key, + request.body.accessKeyId, + request.body.secretAccessKey, request.identity._id.toString, request.identity._organization.toString) ) ?~> "create.failed" } yield Ok(Json.toJson(_id)) } + def createGoogleServiceAccountCredential: Action[GoogleServiceAccountCredentialParameters] = + sil.SecuredAction.async(validateJson[GoogleServiceAccountCredentialParameters]) { implicit request => + val _id = ObjectId.generate + for { + _ <- bool2Fox(request.identity.isAdmin) ?~> "notAllowed" ~> FORBIDDEN + _ <- credentialDAO.insertOne( + _id, + GoogleServiceAccountCredential(request.body.name, + request.body.secretJson, + request.identity._id.toString, + request.identity._organization.toString) + ) ?~> "create.failed" + } yield Ok(Json.toJson(_id)) + } + } diff --git a/app/models/binary/credential/CredentialDAO.scala b/app/models/binary/credential/CredentialDAO.scala index dc9b553e111..c6893f3b030 100644 --- a/app/models/binary/credential/CredentialDAO.scala +++ b/app/models/binary/credential/CredentialDAO.scala @@ -1,8 +1,15 @@ package models.binary.credential import com.scalableminds.util.tools.Fox -import com.scalableminds.webknossos.datastore.storage.{AnyCredential, HttpBasicAuthCredential, S3AccessKeyCredential} +import com.scalableminds.webknossos.datastore.storage.{ + FileSystemCredential, + GoogleServiceAccountCredential, + HttpBasicAuthCredential, + S3AccessKeyCredential +} import com.scalableminds.webknossos.schema.Tables.{Credentials, CredentialsRow} +import net.liftweb.util.Helpers.tryo +import play.api.libs.json.Json import utils.sql.{SecuredSQLDAO, SqlClient, SqlToken} import utils.ObjectId @@ -42,28 +49,51 @@ class CredentialDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContex r._Organization ) + private def parseAsGoogleServiceAccountCredential(r: CredentialsRow): Fox[GoogleServiceAccountCredential] = + for { + secret <- r.secret.toFox + secretJson <- tryo(Json.parse(secret)).toFox + } yield + GoogleServiceAccountCredential( + r.name, + secretJson, + r._User, + r._Organization + ) + def insertOne(_id: ObjectId, credential: HttpBasicAuthCredential): Fox[Unit] = for { _ <- run(q"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization) - values(${_id}, ${CredentialType.HTTP_Basic_Auth}, ${credential.name}, ${credential.username}, ${credential.password}, ${credential.user}, ${credential.organization})""".asUpdate) + values(${_id}, ${CredentialType.HttpBasicAuth}, ${credential.name}, ${credential.username}, ${credential.password}, ${credential.user}, ${credential.organization})""".asUpdate) } yield () def insertOne(_id: ObjectId, credential: S3AccessKeyCredential): Fox[Unit] = for { _ <- run(q"""insert into webknossos.credentials(_id, type, name, identifier, secret, _user, _organization) - values(${_id}, ${CredentialType.S3_Access_Key}, ${credential.name}, ${credential.keyId}, ${credential.key}, ${credential.user}, ${credential.organization})""".asUpdate) + values(${_id}, ${CredentialType.S3AccessKey}, ${credential.name}, ${credential.accessKeyId}, ${credential.secretAccessKey}, ${credential.user}, ${credential.organization})""".asUpdate) + } yield () + + def insertOne(_id: ObjectId, credential: GoogleServiceAccountCredential): Fox[Unit] = + for { + _ <- run(q"""insert into webknossos.credentials(_id, type, name, secret, _user, _organization) + values(${_id}, ${CredentialType.GoogleServiceAccount}, ${credential.name}, ${credential.secretJson.toString}, ${credential.user}, ${credential.organization})""".asUpdate) } yield () - def findOne(id: ObjectId): Fox[AnyCredential] = + def findOne(id: ObjectId): Fox[FileSystemCredential] = for { r <- run(q"select $columns from webknossos.credentials_ where _id = $id".as[CredentialsRow]) firstRow <- r.headOption.toFox parsed <- parseAnyCredential(firstRow) } yield parsed - private def parseAnyCredential(r: CredentialsRow): Fox[AnyCredential] = - r.`type` match { - case "HTTP_Basic_Auth" => parseAsHttpBasicAuthCredential(r) - case "S3_Access_Key" => parseAsS3AccessKeyCredential(r) - } + private def parseAnyCredential(r: CredentialsRow): Fox[FileSystemCredential] = + for { + typeParsed <- CredentialType.fromString(r.`type`).toFox + parsed <- typeParsed match { + case CredentialType.HttpBasicAuth => parseAsHttpBasicAuthCredential(r) + case CredentialType.S3AccessKey => parseAsS3AccessKeyCredential(r) + case CredentialType.GoogleServiceAccount => parseAsGoogleServiceAccountCredential(r) + case _ => Fox.failure(s"Unknown credential type: ${r.`type`}") + } + } yield parsed } diff --git a/app/models/binary/credential/CredentialService.scala b/app/models/binary/credential/CredentialService.scala index ab206fc362d..89857398ddc 100644 --- a/app/models/binary/credential/CredentialService.scala +++ b/app/models/binary/credential/CredentialService.scala @@ -2,52 +2,59 @@ package models.binary.credential import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.storage.{ + FileSystemCredential, FileSystemsHolder, + GoogleServiceAccountCredential, HttpBasicAuthCredential, S3AccessKeyCredential } +import net.liftweb.util.Helpers.tryo +import play.api.libs.json.Json import utils.ObjectId import java.net.URI import javax.inject.Inject import scala.concurrent.ExecutionContext -class CredentialService @Inject()(credentialDao: CredentialDAO) { +class CredentialService @Inject()(credentialDAO: CredentialDAO) { - def createCredential(uri: URI, - username: Option[String], - password: Option[String], - user: String, - organization: String)(implicit ec: ExecutionContext): Fox[Option[ObjectId]] = { - val scheme = uri.getScheme - scheme match { + def createCredentialOpt(uri: URI, + credentialIdentifier: Option[String], + credentialSecret: Option[String], + userId: ObjectId, + organizationId: ObjectId): Option[FileSystemCredential] = + uri.getScheme match { case FileSystemsHolder.schemeHttps => - username match { - case Some(u) => - val _id = ObjectId.generate - for { - _ <- credentialDao.insertOne( - _id, - HttpBasicAuthCredential(uri.toString, u, password.getOrElse(""), user, organization)) - } yield Some(_id) - case None => Fox.successful(None) - } + credentialIdentifier.map( + username => + HttpBasicAuthCredential(uri.toString, + username, + credentialSecret.getOrElse(""), + userId.toString, + organizationId.toString)) case FileSystemsHolder.schemeS3 => - username match { - case Some(keyId) => - password match { - case Some(secretKey) => - val _id = ObjectId.generate - for { - _ <- credentialDao.insertOne( - _id, - S3AccessKeyCredential(uri.toString, keyId, secretKey, user, organization)) - } yield Some(_id) - case None => Fox.successful(None) - } - case None => Fox.successful(None) + (credentialIdentifier, credentialSecret) match { + case (Some(keyId), Some(secretKey)) => + Some(S3AccessKeyCredential(uri.toString, keyId, secretKey, userId.toString, organizationId.toString)) + case _ => None } + case FileSystemsHolder.schemeGS => + for { + secret <- credentialSecret + secretJson <- tryo(Json.parse(secret)).toOption + } yield GoogleServiceAccountCredential(uri.toString, secretJson, userId.toString, organizationId.toString) } + + def insertOne(credential: FileSystemCredential)(implicit ec: ExecutionContext): Fox[ObjectId] = { + val _id = ObjectId.generate + for { + _ <- credential match { + case c: HttpBasicAuthCredential => credentialDAO.insertOne(_id, c) + case c: S3AccessKeyCredential => credentialDAO.insertOne(_id, c) + case c: GoogleServiceAccountCredential => credentialDAO.insertOne(_id, c) + case _ => Fox.failure("Unknown credential type") + } + } yield _id } } diff --git a/app/models/binary/credential/CredentialType.scala b/app/models/binary/credential/CredentialType.scala index d92b80d662d..f0830648f0c 100644 --- a/app/models/binary/credential/CredentialType.scala +++ b/app/models/binary/credential/CredentialType.scala @@ -5,5 +5,5 @@ import com.scalableminds.util.enumeration.ExtendedEnumeration object CredentialType extends ExtendedEnumeration { type CredentialType = Value - val HTTP_Basic_Auth, S3_Access_Key, HTTP_Token, GCS = Value + val HttpBasicAuth, HttpToken, S3AccessKey, GoogleServiceAccount = Value } diff --git a/app/models/binary/explore/ExploreRemoteLayerService.scala b/app/models/binary/explore/ExploreRemoteLayerService.scala index aa0cfe72a98..baa4c2ca18b 100644 --- a/app/models/binary/explore/ExploreRemoteLayerService.scala +++ b/app/models/binary/explore/ExploreRemoteLayerService.scala @@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.dataformats.zarr._ import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header import com.scalableminds.webknossos.datastore.datareaders.zarr._ import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.storage.FileSystemsHolder +import com.scalableminds.webknossos.datastore.storage.{FileSystemsHolder, RemoteSourceDescriptor} import com.typesafe.scalalogging.LazyLogging import models.binary.credential.CredentialService import models.user.User @@ -23,7 +23,9 @@ import scala.collection.mutable.ListBuffer import scala.concurrent.ExecutionContext import scala.util.Try -case class ExploreRemoteDatasetParameters(remoteUri: String, user: Option[String], password: Option[String]) +case class ExploreRemoteDatasetParameters(remoteUri: String, + credentialIdentifier: Option[String], + credentialSecret: Option[String]) object ExploreRemoteDatasetParameters { implicit val jsonFormat: OFormat[ExploreRemoteDatasetParameters] = Json.format[ExploreRemoteDatasetParameters] @@ -39,8 +41,8 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService) exploredLayersNested <- Fox.serialCombined(urisWithCredentials)( parameters => exploreRemoteLayersForUri(parameters.remoteUri, - parameters.user, - parameters.password, + parameters.credentialIdentifier, + parameters.credentialSecret, reportMutable, requestIdentity)) layersWithVoxelSizes = exploredLayersNested.flatten @@ -138,25 +140,27 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService) private def exploreRemoteLayersForUri( layerUri: String, - user: Option[String], - password: Option[String], + credentialIdentifier: Option[String], + credentialSecret: Option[String], reportMutable: ListBuffer[String], requestingUser: User)(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] = for { - remoteSource <- tryo(RemoteSourceDescriptor(new URI(normalizeUri(layerUri)), user, password)).toFox ?~> s"Received invalid URI: $layerUri" - credentialId <- credentialService.createCredential( - new URI(normalizeUri(layerUri)), - user, - password, - requestingUser._id.toString, - requestingUser._organization.toString) ?~> "Failed to set up remote file system credentaial" - fileSystem <- FileSystemsHolder.getOrCreate(remoteSource).toFox ?~> "Failed to set up remote file system" - remotePath <- tryo(fileSystem.getPath(remoteSource.remotePath)) ?~> "Failed to get remote path" + uri <- tryo(new URI(normalizeUri(layerUri))) ?~> s"Received invalid URI: $layerUri" + credentialOpt = credentialService.createCredentialOpt(uri, + credentialIdentifier, + credentialSecret, + requestingUser._id, + requestingUser._organization) + remoteSource = RemoteSourceDescriptor(uri, credentialOpt) + credentialId <- Fox.runOptional(credentialOpt)(c => credentialService.insertOne(c)) ?~> "remoteFileSystem.credential.insert.failed" + fileSystem <- FileSystemsHolder.getOrCreate(remoteSource) ?~> "remoteFileSystem.setup.failed" + remotePath <- tryo(fileSystem.getPath(FileSystemsHolder.pathFromUri(remoteSource.uri))) ?~> "remoteFileSystem.getPath.failed" layersWithVoxelSizes <- exploreRemoteLayersForRemotePath( remotePath, credentialId.map(_.toString), reportMutable, - List(new ZarrArrayExplorer, new NgffExplorer, new N5ArrayExplorer, new N5MultiscalesExplorer)) + List(new ZarrArrayExplorer, new NgffExplorer, new N5ArrayExplorer, new N5MultiscalesExplorer) + ) } yield layersWithVoxelSizes private def normalizeUri(uri: String): String = diff --git a/app/models/binary/explore/N5ArrayExplorer.scala b/app/models/binary/explore/N5ArrayExplorer.scala index 73a2454f869..a91f0f8e83b 100644 --- a/app/models/binary/explore/N5ArrayExplorer.scala +++ b/app/models/binary/explore/N5ArrayExplorer.scala @@ -22,7 +22,12 @@ class N5ArrayExplorer extends RemoteLayerExplorer { elementClass <- n5Header.elementClass ?~> "failed to read element class from n5 header" guessedAxisOrder = AxisOrder.asZyxFromRank(n5Header.rank) boundingBox <- n5Header.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format" - magLocator = MagLocator(Vec3Int.ones, Some(remotePath.toString), None, Some(guessedAxisOrder), None, credentialId) + magLocator = MagLocator(Vec3Int.ones, + Some(remotePath.toUri.toString), + None, + Some(guessedAxisOrder), + None, + credentialId) layer: N5Layer = if (looksLikeSegmentationLayer(name, elementClass)) { N5SegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None) } else N5DataLayer(name, Category.color, boundingBox, elementClass, List(magLocator)) diff --git a/app/models/binary/explore/N5MultiscalesExplorer.scala b/app/models/binary/explore/N5MultiscalesExplorer.scala index 0598843bd2f..a3a597c5abe 100644 --- a/app/models/binary/explore/N5MultiscalesExplorer.scala +++ b/app/models/binary/explore/N5MultiscalesExplorer.scala @@ -108,7 +108,7 @@ class N5MultiscalesExplorer extends RemoteLayerExplorer with FoxImplicits { elementClass <- n5Header.elementClass ?~> s"failed to read element class from n5 header at $headerPath" boundingBox <- n5Header.boundingBox(axisOrder) ?~> s"failed to read bounding box from n5 header at $headerPath" } yield - MagWithAttributes(MagLocator(mag, Some(magPath.toString), None, Some(axisOrder), None, credentialId), + MagWithAttributes(MagLocator(mag, Some(magPath.toUri.toString), None, Some(axisOrder), None, credentialId), magPath, elementClass, boundingBox) diff --git a/app/models/binary/explore/NgffExplorer.scala b/app/models/binary/explore/NgffExplorer.scala index 0f8e772a421..0c17cc23a49 100644 --- a/app/models/binary/explore/NgffExplorer.scala +++ b/app/models/binary/explore/NgffExplorer.scala @@ -126,10 +126,11 @@ class NgffExplorer extends RemoteLayerExplorer { elementClass <- zarrHeader.elementClass ?~> s"failed to read element class from zarr header at $zarrayPath" boundingBox <- zarrHeader.boundingBox(axisOrder) ?~> s"failed to read bounding box from zarr header at $zarrayPath" } yield - MagWithAttributes(MagLocator(mag, Some(magPath.toString), None, Some(axisOrder), channelIndex, credentialId), - magPath, - elementClass, - boundingBox) + MagWithAttributes( + MagLocator(mag, Some(magPath.toUri.toString), None, Some(axisOrder), channelIndex, credentialId), + magPath, + elementClass, + boundingBox) private def extractAxisOrder(axes: List[NgffAxis]): Fox[AxisOrder] = { def axisMatches(axis: NgffAxis, name: String) = axis.name.toLowerCase == name && axis.`type` == "space" diff --git a/app/models/binary/explore/RemoteLayerExplorer.scala b/app/models/binary/explore/RemoteLayerExplorer.scala index 822a05a81ec..39fd1c164f8 100644 --- a/app/models/binary/explore/RemoteLayerExplorer.scala +++ b/app/models/binary/explore/RemoteLayerExplorer.scala @@ -1,6 +1,7 @@ package models.binary.explore import com.scalableminds.util.geometry.{BoundingBox, Vec3Double} +import com.scalableminds.util.io.ZipIO import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} import com.scalableminds.webknossos.datastore.dataformats.MagLocator import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, ElementClass} @@ -24,8 +25,9 @@ trait RemoteLayerExplorer extends FoxImplicits { protected def parseJsonFromPath[T: Reads](path: Path): Fox[T] = for { - fileAsString <- tryo(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)).toFox ?~> "Failed to read remote file" - parsed <- JsonHelper.parseAndValidateJson[T](fileAsString) ?~> "Failed to parse or validate json against data schema" + fileBytes <- tryo(ZipIO.tryGunzip(Files.readAllBytes(path))) ?~> "dataSet.explore.failed.readFile" + fileAsString <- tryo(new String(fileBytes, StandardCharsets.UTF_8)).toFox ?~> "dataSet.explore.failed.readFile" + parsed <- JsonHelper.parseAndValidateJson[T](fileAsString) } yield parsed protected def looksLikeSegmentationLayer(layerName: String, elementClass: ElementClass.Value): Boolean = diff --git a/app/models/binary/explore/ZarrArrayExplorer.scala b/app/models/binary/explore/ZarrArrayExplorer.scala index 2627ccf9de6..7a2c15eb859 100644 --- a/app/models/binary/explore/ZarrArrayExplorer.scala +++ b/app/models/binary/explore/ZarrArrayExplorer.scala @@ -23,7 +23,12 @@ class ZarrArrayExplorer extends RemoteLayerExplorer { elementClass <- zarrHeader.elementClass ?~> "failed to read element class from zarr header" guessedAxisOrder = AxisOrder.asZyxFromRank(zarrHeader.rank) boundingBox <- zarrHeader.boundingBox(guessedAxisOrder) ?~> "failed to read bounding box from zarr header. Make sure data is in (T/C)ZYX format" - magLocator = MagLocator(Vec3Int.ones, Some(remotePath.toString), None, Some(guessedAxisOrder), None, credentialId) + magLocator = MagLocator(Vec3Int.ones, + Some(remotePath.toUri.toString), + None, + Some(guessedAxisOrder), + None, + credentialId) layer: ZarrLayer = if (looksLikeSegmentationLayer(name, elementClass)) { ZarrSegmentationLayer(name, boundingBox, elementClass, List(magLocator), largestSegmentId = None) } else ZarrDataLayer(name, Category.color, boundingBox, elementClass, List(magLocator)) diff --git a/conf/META-INF/services/java.nio.file.spi.FileSystemProvider b/conf/META-INF/services/java.nio.file.spi.FileSystemProvider deleted file mode 100644 index 6739b09b3c5..00000000000 --- a/conf/META-INF/services/java.nio.file.spi.FileSystemProvider +++ /dev/null @@ -1,3 +0,0 @@ -com.scalableminds.webknossos.datastore.s3fs.S3FileSystemProvider -com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpsFileSystemProvider -com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpFileSystemProvider diff --git a/conf/evolutions/099-rename-credential-types.sql b/conf/evolutions/099-rename-credential-types.sql new file mode 100644 index 00000000000..f17f46e6f72 --- /dev/null +++ b/conf/evolutions/099-rename-credential-types.sql @@ -0,0 +1,24 @@ +START TRANSACTION; + +DROP VIEW webknossos.credentials_; + +ALTER TYPE webknossos.CREDENTIAL_TYPE RENAME TO CREDENTIAL_TYPE_OLD; +CREATE TYPE webknossos.CREDENTIAL_TYPE AS ENUM ('HttpBasicAuth', 'HttpToken', 'S3AccessKey', 'GoogleServiceAccount'); + +ALTER TABLE webknossos.credentials + ALTER COLUMN type TYPE webknossos.CREDENTIAL_TYPE USING + CASE type + WHEN 'HTTP_Basic_Auth'::webknossos.CREDENTIAL_TYPE_OLD THEN 'HttpBasicAuth'::webknossos.CREDENTIAL_TYPE + WHEN 'HTTP_Token'::webknossos.CREDENTIAL_TYPE_OLD THEN 'HttpToken'::webknossos.CREDENTIAL_TYPE + WHEN 'S3_Access_Key'::webknossos.CREDENTIAL_TYPE_OLD THEN 'S3AccessKey'::webknossos.CREDENTIAL_TYPE + WHEN 'GCS'::webknossos.CREDENTIAL_TYPE_OLD THEN 'GoogleServiceAccount'::webknossos.CREDENTIAL_TYPE + END; + +DROP TYPE webknossos.CREDENTIAL_TYPE_OLD; + +CREATE VIEW webknossos.credentials_ as SELECT * FROM webknossos.credentials WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation +SET schemaVersion = 99; + +COMMIT TRANSACTION; diff --git a/conf/evolutions/reversions/099-rename-credential-types.sql b/conf/evolutions/reversions/099-rename-credential-types.sql new file mode 100644 index 00000000000..eb9b5cde054 --- /dev/null +++ b/conf/evolutions/reversions/099-rename-credential-types.sql @@ -0,0 +1,24 @@ +START TRANSACTION; + +DROP VIEW webknossos.credentials_; + +ALTER TYPE webknossos.CREDENTIAL_TYPE RENAME TO CREDENTIAL_TYPE_NEW; +CREATE TYPE webknossos.CREDENTIAL_TYPE AS ENUM ('HTTP_Basic_Auth', 'S3_Access_Key', 'HTTP_Token', 'GCS'); + +ALTER TABLE webknossos.credentials + ALTER COLUMN type TYPE webknossos.CREDENTIAL_TYPE USING + CASE type + WHEN 'HttpBasicAuth'::webknossos.CREDENTIAL_TYPE_NEW THEN 'HTTP_Basic_Auth'::webknossos.CREDENTIAL_TYPE + WHEN 'HttpToken'::webknossos.CREDENTIAL_TYPE_NEW THEN 'HTTP_Token'::webknossos.CREDENTIAL_TYPE + WHEN 'S3AccessKey'::webknossos.CREDENTIAL_TYPE_NEW THEN 'S3_Access_Key'::webknossos.CREDENTIAL_TYPE + WHEN 'GoogleServiceAccount'::webknossos.CREDENTIAL_TYPE_NEW THEN 'GCS'::webknossos.CREDENTIAL_TYPE + END; + +DROP TYPE webknossos.CREDENTIAL_TYPE_NEW; + +CREATE VIEW webknossos.credentials_ as SELECT * FROM webknossos.credentials WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation +SET schemaVersion = 98; + +COMMIT TRANSACTION; diff --git a/conf/messages b/conf/messages index a04496cffc8..ebbb138826e 100644 --- a/conf/messages +++ b/conf/messages @@ -100,6 +100,13 @@ dataSet.upload.validation.failed=Failed to validate Dataset information for uplo dataSet.upload.linkRestricted=Can only link layers of datasets that are either public or allowed to be administrated by your account dataSet.upload.invalidLinkedLayers=Could not link all requested layers dataSet.upload.noFiles=Tried to finish upload with no files. May be a retry of a failed finish request, see previous errors. +dataSet.explore.failed.readFile=Failed to read remote file +dataSet.explore.magDtypeMismatch=Element class must be the same for all mags of a layer. Got {0} + +remoteFileSystem.insert.failed=Failed to store remote file system credential +remoteFileSystem.setup.failed=Failed to set up remote file system +remoteFileSystem.getPath.failed=Failed to get remote path + dataSource.notFound=Datasource not found on datastore server diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 9588dee3fbf..3441b98e9cf 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -272,6 +272,7 @@ GET /shortLinks/byKey/:key # Credentials POST /credentials/httpBasicAuth controllers.CredentialController.createHttpBasicAuthCredential POST /credentials/s3AccessKey controllers.CredentialController.createS3AccessKeyCredential +POST /credentials/googleServiceAccount controllers.CredentialController.createGoogleServiceAccountCredential # Voxelytics POST /voxelytics/workflows controllers.VoxelyticsController.storeWorkflow diff --git a/docs/datasets.md b/docs/datasets.md index 4d11296d239..791c633bfc7 100644 --- a/docs/datasets.md +++ b/docs/datasets.md @@ -51,7 +51,7 @@ WEBKNOSSOS can load several Zarr sources and assemble them into a WEBKNOSSOS dat 1. From the *Datasets* tab in the user dashboard, click the *Add Dataset* button. 2. Select the *Add Remote Zarr Dataset* 3. For each layer, provide some metadata information: - - a URL or domain/collection identifier to locate the dataset on the remote service + - a URL or domain/collection identifier to locate the dataset on the remote service (supported protocols are HTTPS, Amazon S3 and Google Cloud Storage). - authentication credentials for accessing the resources on the remote service (optional) 4. Click the *Add Layer* button 5. WEBKNOSSOS will automatically try to infer as many dataset properties (voxel size, bounding box, etc) as possible and preview a [WEBKNOSSOS `datasource` configuration](./data_formats.md#dataset-metadata-specification) for your to review. diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 3a2e550933d..abd783ad44c 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1528,8 +1528,8 @@ export async function exploreRemoteDataset( data: credentials ? remoteUris.map((uri) => ({ remoteUri: uri.trim(), - user: credentials.username, - password: credentials.pass, + credentialIdentifier: credentials.username, + credentialSecret: credentials.pass, })) : remoteUris.map((uri) => ({ remoteUri: uri.trim() })), }); diff --git a/frontend/javascripts/admin/dataset/dataset_add_neuroglancer_view.tsx b/frontend/javascripts/admin/dataset/dataset_add_neuroglancer_view.tsx index 8a4ca0ec2da..0512157fde1 100644 --- a/frontend/javascripts/admin/dataset/dataset_add_neuroglancer_view.tsx +++ b/frontend/javascripts/admin/dataset/dataset_add_neuroglancer_view.tsx @@ -1,4 +1,4 @@ -import { Form, Input, Button, Col, Row, Upload } from "antd"; +import { Form, Input, Button, Col, Row, Upload, UploadFile } from "antd"; import { UnlockOutlined } from "@ant-design/icons"; import { connect } from "react-redux"; import React, { useState } from "react"; @@ -17,6 +17,7 @@ import { DatastoreFormItem, } from "admin/dataset/dataset_components"; import { readFileAsText } from "libs/read_file"; +import { RcFile, UploadChangeParam } from "antd/lib/upload"; const FormItem = Form.Item; type OwnProps = { datastores: Array; @@ -26,9 +27,73 @@ type StateProps = { activeUser: APIUser | null | undefined; }; type Props = OwnProps & StateProps; -type FileList = Array<{ - originFileObj: File; -}>; + +export type FileList = UploadFile[]; + +export const parseCredentials = async (file: RcFile | undefined): Promise => { + if (!file) { + return null; + } + const jsonString = await readFileAsText(file); + try { + return JSON.parse(jsonString); + } catch (_exception) { + Toast.error("Cannot parse credentials as valid JSON. Ignoring credentials file."); + return null; + } +}; + +export function GoogleAuthFormItem({ + fileList, + handleChange, + showOptionalHint, +}: { + fileList: FileList; + handleChange: (arg: UploadChangeParam>) => void; + showOptionalHint?: boolean; +}) { + return ( + + Google{Unicode.NonBreakingSpace} + + Service Account + + {Unicode.NonBreakingSpace}Key {showOptionalHint && "(Optional)"} + + } + hasFeedback + > + false} + > +

+ +

+

+ Click or Drag your Google Cloud Authentication File to this Area to Upload +

+

+ This is only needed if the dataset is located in a non-public Google Cloud Storage bucket +

+
+
+ ); +} function DatasetAddNeuroglancerView({ datastores, onAdded, activeUser }: Props) { const [fileList, setFileList] = useState([]); @@ -66,18 +131,12 @@ function DatasetAddNeuroglancerView({ datastores, onAdded, activeUser }: Props) return config; } - // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'info' implicitly has an 'any' type. - const handleChange = (info) => { + const handleChange = (info: UploadChangeParam>) => { // Restrict the upload list to the latest file const newFileList = info.fileList.slice(-1); setFileList(newFileList); }; - const parseCredentials = async (file: File) => { - const jsonString = await readFileAsText(file); - return JSON.parse(jsonString); - }; - // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'formValues' implicitly has an 'any' typ... Remove this comment to see the full error message async function handleSubmit(formValues) { if (activeUser == null) return; @@ -92,7 +151,7 @@ function DatasetAddNeuroglancerView({ datastores, onAdded, activeUser }: Props) })); const credentials = - fileList.length > 0 ? await parseCredentials(fileList[0].originFileObj) : null; + fileList.length > 0 ? await parseCredentials(fileList[0]?.originFileObj) : null; const datasetConfig = { neuroglancer: { [activeUser.organization]: { @@ -165,47 +224,7 @@ function DatasetAddNeuroglancerView({ datastores, onAdded, activeUser }: Props) > - - Google{Unicode.NonBreakingSpace} - - Service Account - - {Unicode.NonBreakingSpace}Key (Optional) - - } - hasFeedback - > - false} - > -

- -

-

- Click or Drag your Google Cloud Authentication File to this Area to Upload -

-

- This is only needed if the dataset is located in a non-public Google Cloud Storage - bucket -

-
-
+ setDatasourceConfigStr("")} + onClick={() => { + setDatasourceConfigStr(""); + form.resetFields(); + }} > Reset @@ -303,17 +312,26 @@ function AddZarrLayer({ const [showCredentialsFields, setShowCredentialsFields] = useState(false); const [usernameOrAccessKey, setUsernameOrAccessKey] = useState(""); const [passwordOrSecretKey, setPasswordOrSecretKey] = useState(""); - const [selectedProtocol, setSelectedProtocol] = useState<"s3" | "https">("https"); + const [selectedProtocol, setSelectedProtocol] = useState<"s3" | "https" | "gs">("https"); + const [fileList, setFileList] = useState([]); + + const handleChange = (info: UploadChangeParam>) => { + // Restrict the upload list to the latest file + const newFileList = info.fileList.slice(-1); + setFileList(newFileList); + }; function validateUrls(userInput: string) { - if ( - (userInput.indexOf("https://") === 0 && userInput.indexOf("s3://") !== 0) || - (userInput.indexOf("https://") !== 0 && userInput.indexOf("s3://") === 0) - ) { - setSelectedProtocol(userInput.indexOf("https://") === 0 ? "https" : "s3"); - setShowCredentialsFields(userInput.indexOf("s3://") === 0); + if (userInput.startsWith("https://")) { + setSelectedProtocol("https"); + } else if (userInput.startsWith("s3://")) { + setSelectedProtocol("s3"); + } else if (userInput.startsWith("gs://")) { + setSelectedProtocol("gs"); } else { - throw new Error("Dataset URL must employ either the https:// or s3:// protocol."); + throw new Error( + "Dataset URL must employ one of the following protocols: https://, s3:// or gs://", + ); } } @@ -327,13 +345,28 @@ function AddZarrLayer({ syncDataSourceFields(form, dataSourceEditMode === "simple" ? "advanced" : "simple"); const datasourceConfigStr = form.getFieldValue("dataSourceJson"); - const { dataSource: newDataSource, report } = - !usernameOrAccessKey || !passwordOrSecretKey - ? await exploreRemoteDataset([datasourceUrl]) - : await exploreRemoteDataset([datasourceUrl], { + const { dataSource: newDataSource, report } = await (async () => { + if (showCredentialsFields) { + if (selectedProtocol === "gs") { + const credentials = + fileList.length > 0 ? await parseCredentials(fileList[0]?.originFileObj) : null; + if (credentials) { + return exploreRemoteDataset([datasourceUrl], { + username: "", + pass: JSON.stringify(credentials), + }); + } else { + // Fall through to exploreRemoteDataset without parameters + } + } else if (usernameOrAccessKey && passwordOrSecretKey) { + return exploreRemoteDataset([datasourceUrl], { username: usernameOrAccessKey, pass: passwordOrSecretKey, }); + } + } + return exploreRemoteDataset([datasourceUrl]); + })(); setExploreLog(report); if (!newDataSource) { Toast.error( @@ -373,12 +406,13 @@ function AddZarrLayer({ <> Please enter a URL that points to the Zarr or N5 data you would like to import. If necessary, specify the credentials for the dataset. For datasets with multiple layers, e.g. raw - microscopy and segmentattion data, please add them separately with the ”Add Layer” button + microscopy and segmentation data, please add them separately with the ”Add Layer” button below. Once you have approved of the resulting datasource you can import it. {showCredentialsFields ? ( - - - - setUsernameOrAccessKey(e.target.value)} - /> - - - - - setPasswordOrSecretKey(e.target.value)} - /> - - - + selectedProtocol === "gs" ? ( + + ) : ( + + + + setUsernameOrAccessKey(e.target.value)} + /> + + + + + setPasswordOrSecretKey(e.target.value)} + /> + + + + ) ) : null} {exploreLog ? ( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4ed7d9f176b..584168cb62b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -53,6 +53,8 @@ object Dependencies { private val jackson = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.7" private val commonsCompress = "org.apache.commons" % "commons-compress" % "1.21" private val jwt = "com.github.jwt-scala" %% "jwt-play-json" % "9.1.1" + private val googleCloudStorage = "com.google.cloud" % "google-cloud-storage" % "2.13.1" + private val googleCloudStorageNio = "com.google.cloud" % "google-cloud-nio" % "0.123.28" private val sql = Seq( "com.typesafe.slick" %% "slick" % "3.3.3", @@ -102,7 +104,9 @@ object Dependencies { tika, jblosc, scalajHttp, - commonsCompress + commonsCompress, + googleCloudStorage, + googleCloudStorageNio ) val webknossosTracingstoreDependencies: Seq[ModuleID] = Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index d92f3661d6a..eb9239e3231 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -20,4 +20,4 @@ addSbtPlugin("com.sksamuel.scapegoat" %% "sbt-scapegoat" % "1.1.1") addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "3.1.3") //protocol buffers -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.3" +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.12" diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index 10da18783f1..e5e510fc19c 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -19,7 +19,7 @@ START TRANSACTION; CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(98); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(99); COMMIT TRANSACTION; @@ -453,7 +453,7 @@ CREATE TABLE webknossos.shortLinks( longLink Text NOT NULL ); -CREATE TYPE webknossos.CREDENTIAL_TYPE AS ENUM ('HTTP_Basic_Auth', 'S3_Access_Key', 'HTTP_Token', 'GCS'); +CREATE TYPE webknossos.CREDENTIAL_TYPE AS ENUM ('HttpBasicAuth', 'HttpToken', 'S3AccessKey', 'GoogleServiceAccount'); CREATE TABLE webknossos.credentials( _id CHAR(24) PRIMARY KEY, type webknossos.CREDENTIAL_TYPE NOT NULL, diff --git a/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala b/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala index 678c779d545..68f66e3320b 100644 --- a/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala +++ b/util/src/main/scala/com/scalableminds/util/io/ZipIO.scala @@ -145,6 +145,27 @@ object ZipIO extends LazyLogging { } } + def tryGunzip(possiblyCompressed: Array[Byte]): Array[Byte] = + tryo(gunzip(possiblyCompressed)).toOption.getOrElse(possiblyCompressed) + + def gunzip(compressed: Array[Byte]): Array[Byte] = { + val is = new GZIPInputStream(new ByteArrayInputStream(compressed)) + val os = new ByteArrayOutputStream() + try { + val buffer = new Array[Byte](1024) + var len = 0 + do { + len = is.read(buffer) + if (len > 0) + os.write(buffer, 0, len) + } while (len > 0) + os.toByteArray + } finally { + is.close() + os.close() + } + } + def zipToTempFile(files: List[File]): File = { val outfile = File.createTempFile("data", System.nanoTime().toString + ".zip") val zip: OpenZip = startZip(new FileOutputStream(outfile)) diff --git a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala index c63ebbf13dc..9cb37660301 100644 --- a/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala +++ b/util/src/main/scala/com/scalableminds/util/tools/JsonHelper.scala @@ -102,7 +102,8 @@ object JsonHelper extends BoxImplicits with LazyLogging { } def parseAndValidateJson[T: Reads](s: String): Box[T] = - tryo(Json.parse(s)).flatMap(parsed => validateJsValue[T](parsed)) + tryo(Json.parse(s)) + .flatMap(parsed => validateJsValue[T](parsed)) ~> "Failed to parse or validate json against data schema" def validateJsValue[T: Reads](o: JsValue): Box[T] = o.validate[T] match { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala index c9374d687c6..f9877b2a40f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala @@ -1,14 +1,13 @@ package com.scalableminds.webknossos.datastore.dataformats import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.dataformats.zarr.RemoteSourceDescriptor import com.scalableminds.webknossos.datastore.models.BucketPosition import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction -import com.scalableminds.webknossos.datastore.storage.{DataCubeCache, FileSystemService, FileSystemsHolder} +import com.scalableminds.webknossos.datastore.storage.{DataCubeCache, FileSystemService} import com.typesafe.scalalogging.LazyLogging import net.liftweb.common.Empty -import java.nio.file.{FileSystem, Path} +import java.nio.file.Path import scala.concurrent.ExecutionContext trait BucketProvider extends FoxImplicits with LazyLogging { @@ -45,14 +44,6 @@ trait BucketProvider extends FoxImplicits with LazyLogging { def bucketStream(version: Option[Long] = None): Iterator[(BucketPosition, Array[Byte])] = Iterator.empty - protected def remotePathFrom(remoteSource: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[Path] = - FileSystemsHolder - .getOrCreate(remoteSource) - .map { fileSystem: FileSystem => - fileSystem.getPath(remoteSource.remotePath) - } - .toFox - protected def localPathFrom(readInstruction: DataReadInstruction, relativeMagPath: String)( implicit ec: ExecutionContext): Fox[Path] = { val magPath = readInstruction.baseDir diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala index fb0703645f1..4510ec41572 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala @@ -1,17 +1,16 @@ package com.scalableminds.webknossos.datastore.dataformats import com.scalableminds.util.geometry.Vec3Int -import com.scalableminds.webknossos.datastore.dataformats.zarr.{FileSystemCredentials} import com.scalableminds.webknossos.datastore.datareaders.AxisOrder import com.scalableminds.webknossos.datastore.models.datasource.ResolutionFormatHelper -import com.scalableminds.webknossos.datastore.storage.FileSystemsHolder +import com.scalableminds.webknossos.datastore.storage.{FileSystemsHolder, LegacyFileSystemCredential} import play.api.libs.json.{Json, OFormat} import java.net.URI case class MagLocator(mag: Vec3Int, path: Option[String], - credentials: Option[FileSystemCredentials], + credentials: Option[LegacyFileSystemCredential], axisOrder: Option[AxisOrder], channelIndex: Option[Int], credentialId: Option[String]) { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala index ee15a7d1076..c164d6d407a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala @@ -46,10 +46,7 @@ class N5BucketProvider(layer: N5Layer, val fileSystemServiceOpt: Option[FileSyst case Some(fileSystemService: FileSystemService) => for { magPath: Path <- if (n5Mag.isRemote) { - for { - remoteSource <- fileSystemService.remoteSourceFor(n5Mag) - remotePath <- remotePathFrom(remoteSource) - } yield remotePath + fileSystemService.remotePathFor(n5Mag) } else localPathFrom(readInstruction, n5Mag.pathWithFallback) cubeHandle <- tryo(onError = e => logError(e))(N5Array.open(magPath, n5Mag.axisOrder, n5Mag.channelIndex)) .map(new N5CubeHandle(_)) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala index 6cc56fb16ef..ec352a48a82 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala @@ -46,10 +46,7 @@ class ZarrBucketProvider(layer: ZarrLayer, val fileSystemServiceOpt: Option[File case Some(fileSystemService: FileSystemService) => for { magPath: Path <- if (zarrMag.isRemote) { - for { - remoteSource <- fileSystemService.remoteSourceFor(zarrMag) - remotePath <- remotePathFrom(remoteSource) - } yield remotePath + fileSystemService.remotePathFor(zarrMag) } else localPathFrom(readInstruction, zarrMag.pathWithFallback) cubeHandle <- tryo(onError = e => logError(e))( ZarrArray.open(magPath, zarrMag.axisOrder, zarrMag.channelIndex)).map(new ZarrCubeHandle(_)) @@ -58,4 +55,5 @@ class ZarrBucketProvider(layer: ZarrLayer, val fileSystemServiceOpt: Option[File } } } + } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrDataLayers.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrDataLayers.scala index ddf779ac4bc..bc6cb0456a0 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrDataLayers.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrDataLayers.scala @@ -7,19 +7,6 @@ import com.scalableminds.webknossos.datastore.models.datasource._ import com.scalableminds.webknossos.datastore.storage.FileSystemService import play.api.libs.json.{Json, OFormat} -import java.net.URI - -case class FileSystemCredentials(user: String, password: Option[String]) - -object FileSystemCredentials { - implicit val jsonFormat: OFormat[FileSystemCredentials] = Json.format[FileSystemCredentials] -} - -case class RemoteSourceDescriptor(uri: URI, user: Option[String], password: Option[String]) { - lazy val remotePath: String = uri.getPath - lazy val credentials: Option[FileSystemCredentials] = user.map(u => FileSystemCredentials(u, password)) -} - trait ZarrLayer extends DataLayer { val dataFormat: DataFormat.Value = DataFormat.zarr diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/FileSystemStore.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/FileSystemStore.scala index 6f7683fc55e..92219bf21bc 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/FileSystemStore.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/FileSystemStore.scala @@ -1,5 +1,6 @@ package com.scalableminds.webknossos.datastore.datareaders +import com.scalableminds.util.io.ZipIO import net.liftweb.util.Helpers.tryo import java.nio.file.{Files, Path} @@ -7,6 +8,6 @@ import java.nio.file.{Files, Path} class FileSystemStore(val internalRoot: Path) { def readBytes(key: String): Option[Array[Byte]] = { val path = internalRoot.resolve(key) - tryo(Files.readAllBytes(path)).toOption + tryo(Files.readAllBytes(path)).toOption.map(ZipIO.tryGunzip) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystem.java b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystem.java index fd83598c1a5..4de5c33c0a7 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystem.java +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystem.java @@ -1,6 +1,7 @@ package com.scalableminds.webknossos.datastore.s3fs; import java.io.IOException; +import java.net.URI; import java.nio.file.FileStore; import java.nio.file.FileSystem; import java.nio.file.Path; @@ -11,6 +12,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.Bucket; +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -21,6 +23,14 @@ */ public class S3FileSystem extends FileSystem implements Comparable { + public static S3FileSystem forUri(URI uri, String keyId, String secretKey) { + return S3FileSystemProvider.forUri(uri, keyId, secretKey); + } + + public static S3FileSystem forUri(URI uri) { + return forUri(uri, null, null); + } + private final S3FileSystemProvider provider; private final String key; private final AmazonS3 client; diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystemProvider.java b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystemProvider.java index 04938ae60cc..dfdd2f03d8b 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystemProvider.java +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3FileSystemProvider.java @@ -37,8 +37,8 @@ /** * Spec: *

- * URI: s3://[endpoint]/{bucket}/{key} If endpoint is missing, it's assumed to - * be the default S3 endpoint (s3.amazonaws.com) + * URI: s3://[user@]{bucket}/{key} + * It's assumed to use the default S3 endpoint (s3.amazonaws.com) *

*

* FileSystem roots: /{bucket}/ @@ -63,8 +63,6 @@ *

*/ public class S3FileSystemProvider extends FileSystemProvider { - - public static final String CHARSET_KEY = "s3fs_charset"; public static final String AMAZON_S3_FACTORY_CLASS = "s3fs_amazon_s3_factory"; private static final ConcurrentMap fileSystems = new ConcurrentHashMap<>(); @@ -72,7 +70,7 @@ public class S3FileSystemProvider extends FileSystemProvider { AmazonS3Factory.PROXY_HOST, AmazonS3Factory.PROXY_PASSWORD, AmazonS3Factory.PROXY_PORT, AmazonS3Factory.PROXY_USERNAME, AmazonS3Factory.PROXY_WORKSTATION, AmazonS3Factory.SOCKET_SEND_BUFFER_SIZE_HINT, AmazonS3Factory.SOCKET_RECEIVE_BUFFER_SIZE_HINT, AmazonS3Factory.SOCKET_TIMEOUT, AmazonS3Factory.USER_AGENT, AMAZON_S3_FACTORY_CLASS, AmazonS3Factory.SIGNER_OVERRIDE, AmazonS3Factory.PATH_STYLE_ACCESS); - private S3Utils s3Utils = new S3Utils(); + private final S3Utils s3Utils = new S3Utils(); private Cache cache = new Cache(); @Override @@ -80,6 +78,25 @@ public String getScheme() { return "s3"; } + public static S3FileSystem forUri(URI uri, String accessKey, String secretKey) { + Properties props = new Properties(); + if (accessKey != null && secretKey != null) { + props.put(AmazonS3Factory.ACCESS_KEY, accessKey); + props.put(AmazonS3Factory.SECRET_KEY, secretKey); + } + URI uriWithNormalizedHost = resolveShortcutHost(uri); + String key = buildFileSystemKey(uriWithNormalizedHost, accessKey); + if (fileSystems.containsKey(key)) { + return fileSystems.get(key); + } + String bucket = S3FileSystemProvider.hostBucketFromUri(uri); + S3FileSystemProvider provider = new S3FileSystemProvider(); + AmazonS3 client = new AmazonS3Factory().getAmazonS3Client(uri, props); + S3FileSystem fileSystem = new S3FileSystem(provider, key, client, bucket); + fileSystems.put(fileSystem.getKey(), fileSystem); + return fileSystem; + } + @Override public FileSystem newFileSystem(URI uri, Map env) { validateUri(uri); @@ -87,7 +104,7 @@ public FileSystem newFileSystem(URI uri, Map env) { Properties props = getProperties(uri, env); validateProperties(props); // try to get the filesystem by the key - String key = getFileSystemKey(uri, props); + String key = S3FileSystemProvider.buildFileSystemKey(uri, (String) props.get(AmazonS3Factory.ACCESS_KEY)); if (fileSystems.containsKey(key)) { throw new FileSystemAlreadyExistsException("File system " + uri.getScheme() + ':' + key + " already exists"); } @@ -119,8 +136,8 @@ private Properties getProperties(URI uri, Map env) { return props; } - private String getFileSystemKey(URI uri) { - return getFileSystemKey(uri, getProperties(uri, null)); + private static String buildFileSystemKey(URI uri) { + return buildFileSystemKey(uri, null); } /** @@ -129,35 +146,34 @@ private String getFileSystemKey(URI uri) { * If uri host is empty then s3.amazonaws.com are used as host * * @param uri URI with the endpoint - * @param props with the access key property + * @param accessKey access key id * @return String */ - protected String getFileSystemKey(URI uri, Properties props) { - // we don`t use uri.getUserInfo and uri.getHost because secret key and access key have special chars - // and dont return the correct strings - String uriString = uri.toString().replace("s3://", ""); - String authority = null; - int authoritySeparator = uriString.indexOf("@"); - - if (authoritySeparator > 0) { - authority = uriString.substring(0, authoritySeparator); - } + protected static String buildFileSystemKey(URI uri, String accessKey) { + // we don`t use uri.getUserInfo and uri.getHost because secret key and access key have special chars + // and dont return the correct strings + String uriString = uri.toString().replace("s3://", ""); + String authority = null; + int authoritySeparator = uriString.indexOf("@"); + + if (authoritySeparator > 0) { + authority = uriString.substring(0, authoritySeparator); + } - if (authority != null) { - String host = uriString.substring(uriString.indexOf("@") + 1, uriString.length()); - int lastPath = host.indexOf("/"); - if (lastPath > -1) { - host = host.substring(0, lastPath); - } - if (host.length() == 0) { - host = Constants.S3_HOSTNAME; - } - return authority + "@" + host; - } else { - String accessKey = (String) props.get(AmazonS3Factory.ACCESS_KEY); - return (accessKey != null ? accessKey + "@" : "") + - (uri.getHost() != null ? uri.getHost() : Constants.S3_HOSTNAME); + if (authority != null) { + String host = uriString.substring(uriString.indexOf("@") + 1, uriString.length()); + int lastPath = host.indexOf("/"); + if (lastPath > -1) { + host = host.substring(0, lastPath); } + if (host.length() == 0) { + host = Constants.S3_HOSTNAME; + } + return authority + "@" + host; + } else { + return (accessKey != null ? accessKey + "@" : "") + + (uri.getHost() != null ? uri.getHost() : Constants.S3_HOSTNAME); + } } protected void validateUri(URI uri) { @@ -261,7 +277,7 @@ public String systemGetEnv(String key) { public FileSystem getFileSystem(URI uri, Map env) { validateUri(uri); Properties props = getProperties(uri, env); - String key = this.getFileSystemKey(uri, props); // s3fs_access_key is part of the key here. + String key = S3FileSystemProvider.buildFileSystemKey(uri, (String) props.get(AmazonS3Factory.ACCESS_KEY)); // s3fs_access_key is part of the key here. if (fileSystems.containsKey(key)) return fileSystems.get(key); return newFileSystem(uri, env); @@ -270,7 +286,7 @@ public FileSystem getFileSystem(URI uri, Map env) { @Override public S3FileSystem getFileSystem(URI uri) { validateUri(uri); - String key = this.getFileSystemKey(uri); + String key = S3FileSystemProvider.buildFileSystemKey(uri); if (fileSystems.containsKey(key)) { return fileSystems.get(key); } else { @@ -538,8 +554,6 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption.. throw new UnsupportedOperationException(); } - // ~~ - /** * Create the fileSystem * @@ -547,13 +561,14 @@ public void setAttribute(Path path, String attribute, Object value, LinkOption.. * @param props Properties * @return S3FileSystem never null */ - public S3FileSystem createFileSystem(URI uri, Properties props) { - URI uriWithNormalizedHost = resolveShortcutHost(uri); + private S3FileSystem createFileSystem(URI uri, Properties props) { + URI uriWithNormalizedHost = S3FileSystemProvider.resolveShortcutHost(uri); String bucket = hostBucketFromUri(uri); - return new S3FileSystem(this, getFileSystemKey(uriWithNormalizedHost, props), getAmazonS3(uriWithNormalizedHost, props), bucket); + String key = S3FileSystemProvider.buildFileSystemKey(uri, (String) props.get(AmazonS3Factory.ACCESS_KEY)); + return new S3FileSystem(this, key, getAmazonS3(uriWithNormalizedHost, props), bucket); } - private URI resolveShortcutHost(URI uri) { + public static URI resolveShortcutHost(URI uri) { String host = uri.getHost(); String newHost = host; String bucketPrefix = ""; @@ -572,7 +587,7 @@ private URI resolveShortcutHost(URI uri) { } - private String hostBucketFromUri(URI uri) { + private static String hostBucketFromUri(URI uri) { String host = uri.getHost(); if (!host.contains(".")) { // assume host is omitted from uri, shortcut form s3://bucket/key return host; diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala index 99312cbdc51..682ec8e3219 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala @@ -12,7 +12,7 @@ import com.scalableminds.webknossos.datastore.models.annotation.AnnotationSource import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.datastore.models.datasource.inbox.InboxDataSourceLike import com.scalableminds.webknossos.datastore.rpc.RPC -import com.scalableminds.webknossos.datastore.storage.AnyCredential +import com.scalableminds.webknossos.datastore.storage.FileSystemCredential import com.typesafe.scalalogging.LazyLogging import play.api.inject.ApplicationLifecycle import play.api.libs.json.{Json, OFormat} @@ -150,10 +150,17 @@ class DSRemoteWebKnossosClient @Inject()( .getWithJsonResponse[AnnotationSource] ) - def findCredential(credentialId: String): Fox[AnyCredential] = - rpc(s"$webKnossosUri/api/datastores/$dataStoreName/findCredential") - .addQueryString("credentialId" -> credentialId) - .addQueryString("key" -> dataStoreKey) - .silent - .getWithJsonResponse[AnyCredential] + private lazy val credentialCache: AlfuFoxCache[String, FileSystemCredential] = + AlfuFoxCache(timeToLive = 5 seconds, timeToIdle = 5 seconds) + + def getCredential(credentialId: String): Fox[FileSystemCredential] = + credentialCache.getOrLoad( + credentialId, + _ => + rpc(s"$webKnossosUri/api/datastores/$dataStoreName/findCredential") + .addQueryString("credentialId" -> credentialId) + .addQueryString("key" -> dataStoreKey) + .silent + .getWithJsonResponse[FileSystemCredential] + ) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemCredentials.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemCredentials.scala index d7c9e1700d1..13bec152159 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemCredentials.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemCredentials.scala @@ -1,23 +1,50 @@ package com.scalableminds.webknossos.datastore.storage -import play.api.libs.json.{Json, OFormat} +import play.api.libs.json.{JsValue, Json, OFormat} -sealed trait AnyCredential +sealed trait FileSystemCredential -object AnyCredential { - implicit val jsonFormat: OFormat[AnyCredential] = Json.format[AnyCredential] +object FileSystemCredential { + implicit val jsonFormat: OFormat[FileSystemCredential] = Json.format[FileSystemCredential] } case class HttpBasicAuthCredential(name: String, username: String, password: String, user: String, organization: String) - extends AnyCredential + extends FileSystemCredential object HttpBasicAuthCredential { implicit val jsonFormat: OFormat[HttpBasicAuthCredential] = Json.format[HttpBasicAuthCredential] } -case class S3AccessKeyCredential(name: String, keyId: String, key: String, user: String, organization: String) - extends AnyCredential +case class S3AccessKeyCredential(name: String, + accessKeyId: String, + secretAccessKey: String, + user: String, + organization: String) + extends FileSystemCredential object S3AccessKeyCredential { implicit val jsonFormat: OFormat[S3AccessKeyCredential] = Json.format[S3AccessKeyCredential] } + +case class GoogleServiceAccountCredential(name: String, secretJson: JsValue, user: String, organization: String) + extends FileSystemCredential + +object GoogleServiceAccountCredential { + implicit val jsonFormat: OFormat[GoogleServiceAccountCredential] = Json.format[GoogleServiceAccountCredential] +} + +case class LegacyFileSystemCredential(user: String, password: Option[String]) extends FileSystemCredential { + def toBasicAuth: HttpBasicAuthCredential = + HttpBasicAuthCredential(name = "", username = user, password = password.getOrElse(""), user = "", organization = "") + + def toS3AccessKey: S3AccessKeyCredential = + S3AccessKeyCredential(name = "", + accessKeyId = user, + secretAccessKey = password.getOrElse(""), + user = "", + organization = "") +} + +object LegacyFileSystemCredential { + implicit val jsonFormat: OFormat[LegacyFileSystemCredential] = Json.format[LegacyFileSystemCredential] +} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemService.scala index 4b01ec82ad5..22028faed25 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemService.scala @@ -2,34 +2,34 @@ package com.scalableminds.webknossos.datastore.storage import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.dataformats.MagLocator -import com.scalableminds.webknossos.datastore.dataformats.zarr.RemoteSourceDescriptor import com.scalableminds.webknossos.datastore.services.DSRemoteWebKnossosClient +import net.liftweb.util.Helpers.tryo import java.net.URI +import java.nio.file.Path import javax.inject.Inject import scala.concurrent.ExecutionContext +case class RemoteSourceDescriptor(uri: URI, credential: Option[FileSystemCredential]) + class FileSystemService @Inject()(dSRemoteWebKnossosClient: DSRemoteWebKnossosClient) { - private def remoteSourceDescriptorFromCredential(uri: URI, credential: AnyCredential): RemoteSourceDescriptor = - credential match { - case HttpBasicAuthCredential(name, username, password, _, _) => - RemoteSourceDescriptor(uri, Some(username), Some(password)) - case S3AccessKeyCredential(name, keyId, key, _, _) => RemoteSourceDescriptor(uri, Some(keyId), Some(key)) - } + def remotePathFor(magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[Path] = + for { + credentialBox <- credentialFor(magLocator: MagLocator).futureBox + remoteSource = RemoteSourceDescriptor(magLocator.uri, credentialBox.toOption) + fileSystem <- FileSystemsHolder.getOrCreate(remoteSource) ?~> "remoteFileSystem.setup.failed" + remotePath <- tryo(fileSystem.getPath(FileSystemsHolder.pathFromUri(remoteSource.uri))) + } yield remotePath - def remoteSourceFor(magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[RemoteSourceDescriptor] = + private def credentialFor(magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[FileSystemCredential] = magLocator.credentialId match { case Some(credentialId) => - for { - credential <- dSRemoteWebKnossosClient.findCredential(credentialId) - descriptor = remoteSourceDescriptorFromCredential(magLocator.uri, credential) - } yield descriptor + dSRemoteWebKnossosClient.getCredential(credentialId) case None => magLocator.credentials match { - case Some(credentials) => - Fox.successful(RemoteSourceDescriptor(magLocator.uri, Some(credentials.user), credentials.password)) - case None => Fox.successful(RemoteSourceDescriptor(magLocator.uri, None, None)) + case Some(credential) => Fox.successful(credential) + case None => Fox.empty } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemsHolder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemsHolder.scala index fa14bb9cc89..fa98e5c00af 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemsHolder.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/FileSystemsHolder.scala @@ -1,99 +1,100 @@ package com.scalableminds.webknossos.datastore.storage -import java.lang.Thread.currentThread -import java.net.URI -import java.nio.file.spi.FileSystemProvider -import java.nio.file.{FileSystem, FileSystemAlreadyExistsException, FileSystems} -import java.util.ServiceLoader -import com.google.common.collect.ImmutableMap -import com.scalableminds.util.cache.LRUConcurrentCache -import com.scalableminds.webknossos.datastore.dataformats.zarr.RemoteSourceDescriptor -import com.scalableminds.webknossos.datastore.s3fs.AmazonS3Factory +import com.google.auth.oauth2.ServiceAccountCredentials +import com.google.cloud.storage.StorageOptions +import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem} +import com.scalableminds.util.cache.AlfuFoxCache +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.s3fs.{S3FileSystem, S3FileSystemProvider} +import com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpsFileSystem import com.typesafe.scalalogging.LazyLogging +import net.liftweb.common.Full -import scala.collection.JavaConverters._ - -class FileSystemsCache(val maxEntries: Int) extends LRUConcurrentCache[RemoteSourceDescriptor, FileSystem] -class FileSystemsProvidersCache(val maxEntries: Int) extends LRUConcurrentCache[String, FileSystemProvider] +import java.io.ByteArrayInputStream +import java.net.URI +import java.nio.file.FileSystem +import scala.concurrent.ExecutionContext object FileSystemsHolder extends LazyLogging { val schemeS3: String = "s3" val schemeHttps: String = "https" val schemeHttp: String = "http" + val schemeGS: String = "gs" - private val fileSystemsCache = new FileSystemsCache(maxEntries = 100) - private val fileSystemsProvidersCache = new FileSystemsProvidersCache(maxEntries = 100) + private val fileSystemsCache: AlfuFoxCache[RemoteSourceDescriptor, FileSystem] = + AlfuFoxCache(maxEntries = 100) def isSupportedRemoteScheme(uriScheme: String): Boolean = - List(schemeS3, schemeHttps, schemeHttp).contains(uriScheme) - - def getOrCreate(remoteSource: RemoteSourceDescriptor): Option[FileSystem] = - fileSystemsCache.getOrLoadAndPutOptional(remoteSource)(loadFromProvider) + List(schemeS3, schemeHttps, schemeHttp, schemeGS).contains(uriScheme) - private def loadFromProvider(remoteSource: RemoteSourceDescriptor): Option[FileSystem] = { - /* - * The FileSystemProviders can have their own cache for file systems. - * Those will error on create if the file system already exists. - * Quirk: They include the user name in the key. - * This is not supported for newFileSystem but is for getFileSystem. - * Conversely, getFileSystem cannot be called with the credentials env. - * Hence this has to be called in two different ways here. - */ - val uriWithPath = remoteSource.uri - val uri = baseUri(uriWithPath) - val uriWithUser = insertUserName(uri, remoteSource) - - val scheme = uri.getScheme - val credentialsEnv = makeCredentialsEnv(remoteSource, scheme) + def getOrCreate(remoteSource: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[FileSystem] = + fileSystemsCache.getOrLoad(remoteSource, create) + def create(remoteSource: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[FileSystem] = { + val scheme = remoteSource.uri.getScheme try { - Some(FileSystems.newFileSystem(uri, credentialsEnv, currentThread().getContextClassLoader)) + val fs: FileSystem = if (scheme == schemeGS) { + getGoogleCloudStorageFileSystem(remoteSource) + } else if (scheme == schemeS3) { + getAmazonS3FileSystem(remoteSource) + } else if (scheme == schemeHttps || scheme == schemeHttp) { + getHttpsFileSystem(remoteSource) + } else { + throw new Exception(s"Unknown file system scheme $scheme") + } + logger.info(s"Successfully created file system for ${remoteSource.uri.toString}") + Fox.successful(fs) } catch { - case _: FileSystemAlreadyExistsException => - try { - findProviderWithCache(uri.getScheme).map(_.getFileSystem(uriWithUser)) - } catch { - case e2: Exception => - logger.error(s"getFileSytem errored for ${uriWithUser.toString}:", e2) - None - } + case e: Exception => + val msg = s"get file system errored for ${remoteSource.uri.toString}:" + logger.error(msg, e) + Fox.failure(msg, Full(e)) } } - private def insertUserName(uri: URI, remoteSource: RemoteSourceDescriptor): URI = - remoteSource.user.map { user => - new URI(uri.getScheme, user, uri.getHost, uri.getPort, uri.getPath, uri.getQuery, uri.getFragment) - }.getOrElse(uri) - - private def baseUri(uri: URI): URI = - new URI(uri.getScheme, uri.getUserInfo, uri.getHost, uri.getPort, null, null, null) - - private def makeCredentialsEnv(remoteSource: RemoteSourceDescriptor, scheme: String): ImmutableMap[String, Any] = - (for { - user <- remoteSource.user - password <- remoteSource.password - } yield { - if (scheme == schemeS3) { - ImmutableMap - .builder[String, Any] - .put(AmazonS3Factory.ACCESS_KEY, user) - .put(AmazonS3Factory.SECRET_KEY, password) - .build - } else if (scheme == schemeHttps || scheme == schemeHttp) { - ImmutableMap.builder[String, Any].put("user", user).put("password", password).build - } else emptyEnv - }).getOrElse(emptyEnv) + private def getGoogleCloudStorageFileSystem(remoteSource: RemoteSourceDescriptor): FileSystem = { + val bucketName = remoteSource.uri.getHost + val storageOptions = buildCredentialStorageOptions(remoteSource) + CloudStorageFileSystem.forBucket(bucketName, CloudStorageConfiguration.DEFAULT, storageOptions) + } - private def emptyEnv: ImmutableMap[String, Any] = ImmutableMap.builder[String, Any].build() + private def getAmazonS3FileSystem(remoteSource: RemoteSourceDescriptor): FileSystem = + remoteSource.credential match { + case Some(credential: S3AccessKeyCredential) => + S3FileSystem.forUri(remoteSource.uri, credential.accessKeyId, credential.secretAccessKey) + case Some(credential: LegacyFileSystemCredential) => + S3FileSystem.forUri(remoteSource.uri, + credential.toS3AccessKey.accessKeyId, + credential.toS3AccessKey.secretAccessKey) + case _ => + S3FileSystem.forUri(remoteSource.uri) + } - private def findProviderWithCache(scheme: String): Option[FileSystemProvider] = - fileSystemsProvidersCache.getOrLoadAndPutOptional(scheme: String)(findProvider) + private def getHttpsFileSystem(remoteSource: RemoteSourceDescriptor): FileSystem = + remoteSource.credential match { + case Some(credential: HttpBasicAuthCredential) => + HttpsFileSystem.forUri(remoteSource.uri, Some(credential)) + case Some(credential: LegacyFileSystemCredential) => + HttpsFileSystem.forUri(remoteSource.uri, Some(credential.toBasicAuth)) + case _ => + HttpsFileSystem.forUri(remoteSource.uri) + } - private def findProvider(scheme: String): Option[FileSystemProvider] = { - val providersIterator = - ServiceLoader.load(classOf[FileSystemProvider], currentThread().getContextClassLoader).iterator().asScala - providersIterator.find(p => p.getScheme.equalsIgnoreCase(scheme)) - } + private def buildCredentialStorageOptions(remoteSource: RemoteSourceDescriptor) = + remoteSource.credential match { + case Some(credential: GoogleServiceAccountCredential) => + StorageOptions + .newBuilder() + .setCredentials( + ServiceAccountCredentials.fromStream(new ByteArrayInputStream(credential.secretJson.toString.getBytes))) + .build() + case _ => StorageOptions.newBuilder().build() + } + def pathFromUri(uri: URI): String = + if (uri.getScheme == schemeS3) { + // There are several s3 uri styles. Normalize, then drop bucket name (and handle leading slash) + "/" + S3FileSystemProvider.resolveShortcutHost(uri).getPath.split("/").drop(2).mkString("/") + } else uri.getPath } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpFileSystemProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpFileSystemProvider.scala deleted file mode 100644 index b5412653c7f..00000000000 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpFileSystemProvider.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.scalableminds.webknossos.datastore.storage.httpsfilesystem - -class HttpFileSystemProvider extends HttpsFileSystemProvider { - override def getScheme: String = "http" -} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsBasicAuthCredentials.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsBasicAuthCredentials.scala deleted file mode 100644 index b74f5372517..00000000000 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsBasicAuthCredentials.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.scalableminds.webknossos.datastore.storage.httpsfilesystem - -import java.util - -import scala.jdk.CollectionConverters.mapAsScalaMapConverter - -case class HttpsBasicAuthCredentials(user: String, password: String) - -object HttpsBasicAuthCredentials { - def fromEnvMap(env: util.Map[String, _]): Option[HttpsBasicAuthCredentials] = { - val envAsScala = env.asScala - val userOpt = envAsScala.get("user") - val passwordOpt = envAsScala.get("password") - for { - user <- userOpt - password <- passwordOpt - } yield HttpsBasicAuthCredentials(user.asInstanceOf[String], password.asInstanceOf[String]) - } -} diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystem.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystem.scala index d53aa373ff1..39c4faa3594 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystem.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystem.scala @@ -1,14 +1,31 @@ package com.scalableminds.webknossos.datastore.storage.httpsfilesystem +import com.scalableminds.webknossos.datastore.storage.HttpBasicAuthCredential + import java.net.URI import java.nio.file._ import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider import java.{lang, util} +object HttpsFileSystem { + def forUri(uri: URI, credential: Option[HttpBasicAuthCredential] = None): HttpsFileSystem = { + + val key = HttpsFileSystemProvider.fileSystemKey(uri, None) + if (HttpsFileSystemProvider.fileSystems.containsKey(key)) { + HttpsFileSystemProvider.fileSystems.get(key) + } else { + val fileSystem = new HttpsFileSystem(new HttpsFileSystemProvider, uri, credential) + HttpsFileSystemProvider.fileSystems.put(key, fileSystem) + fileSystem + } + } + +} + class HttpsFileSystem(provider: HttpsFileSystemProvider, uri: URI, - basicAuthCredentials: Option[HttpsBasicAuthCredentials] = None) + basicAuthCredential: Option[HttpBasicAuthCredential] = None) extends FileSystem { override def provider(): FileSystemProvider = provider @@ -35,7 +52,7 @@ class HttpsFileSystem(provider: HttpsFileSystemProvider, override def newWatchService(): WatchService = ??? - def getKey: String = provider.fileSystemKey(uri, basicAuthCredentials) + def getKey: String = HttpsFileSystemProvider.fileSystemKey(uri, basicAuthCredential) - def getBasicAuthCredentials: Option[HttpsBasicAuthCredentials] = basicAuthCredentials + def getBasicAuthCredential: Option[HttpBasicAuthCredential] = basicAuthCredential } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystemProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystemProvider.scala index 80926c4fcdc..f33a098749f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystemProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsFileSystemProvider.scala @@ -7,48 +7,29 @@ import java.nio.file.spi.FileSystemProvider import java.nio.file._ import java.util import java.util.concurrent.ConcurrentHashMap - -import com.google.common.collect.ImmutableMap +import com.scalableminds.webknossos.datastore.storage.HttpBasicAuthCredential import com.typesafe.scalalogging.LazyLogging -class HttpsFileSystemProvider extends FileSystemProvider with LazyLogging { - protected val fileSystems: ConcurrentHashMap[String, HttpsFileSystem] = new ConcurrentHashMap[String, HttpsFileSystem] - - override def getScheme: String = "https" - - override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = { - if (uri.getUserInfo != null && uri.getUserInfo.nonEmpty) { - throw new Exception("Username was supplied in uri, should be in env instead.") - } - val basicAuthCredentials = HttpsBasicAuthCredentials.fromEnvMap(env) - val key = fileSystemKey(uri, basicAuthCredentials) - if (fileSystems.containsKey(key)) { - throw new FileSystemAlreadyExistsException("File system " + key + " already exists") - } - - val fileSystem = new HttpsFileSystem(provider = this, uri = uri, basicAuthCredentials = basicAuthCredentials) - - fileSystems.put(fileSystem.getKey, fileSystem) - - fileSystem - } +object HttpsFileSystemProvider { + val fileSystems: ConcurrentHashMap[String, HttpsFileSystem] = new ConcurrentHashMap[String, HttpsFileSystem] - def fileSystemKey(uri: URI, basicAuthCredentials: Option[HttpsBasicAuthCredentials]): String = { + def fileSystemKey(uri: URI, basicAuthCredentials: Option[HttpBasicAuthCredential]): String = { val uriWithUser = basicAuthCredentials.map { c => new URI(uri.getScheme, c.user, uri.getHost, uri.getPort, uri.getPath, uri.getQuery, uri.getFragment) }.getOrElse(uri) uriWithUser.toString } +} - override def getFileSystem(uri: URI): FileSystem = { - val key = fileSystemKey(uri, None) - if (fileSystems.containsKey(key)) { - fileSystems.get(key) - } else this.newFileSystem(uri, ImmutableMap.builder[String, Any].build()) - } +class HttpsFileSystemProvider extends FileSystemProvider with LazyLogging { + + override def getScheme: String = "https" // Note that it will also handle http if called with one + + override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = ??? + + override def getFileSystem(uri: URI): FileSystem = ??? - override def getPath(uri: URI): Path = - getFileSystem(uri).getPath(uri.getPath) + override def getPath(uri: URI): Path = ??? override def newByteChannel(path: Path, openOptions: util.Set[_ <: OpenOption], diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsPath.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsPath.scala index a7c8df0ed27..c4a66de3570 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsPath.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsPath.scala @@ -1,5 +1,7 @@ package com.scalableminds.webknossos.datastore.storage.httpsfilesystem +import com.scalableminds.webknossos.datastore.storage.HttpBasicAuthCredential + import java.io.File import java.net.URI import java.nio.file.{FileSystem, LinkOption, Path, WatchEvent, WatchKey, WatchService} @@ -65,5 +67,5 @@ class HttpsPath(uri: URI, fileSystem: HttpsFileSystem) extends Path { override def toString: String = uri.toString - def getBasicAuthCredentials: Option[HttpsBasicAuthCredentials] = fileSystem.getBasicAuthCredentials + def getBasicAuthCredential: Option[HttpBasicAuthCredential] = fileSystem.getBasicAuthCredential } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsSeekableByteChannel.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsSeekableByteChannel.scala index f6e113d23b8..ffc55ddd46f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsSeekableByteChannel.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/httpsfilesystem/HttpsSeekableByteChannel.scala @@ -16,15 +16,16 @@ import scala.concurrent.duration.DurationInt class HttpsSeekableByteChannel(path: HttpsPath, openOptions: util.Set[_ <: OpenOption]) extends SeekableByteChannel { private var _position: Long = 0L + private val uri: URI = path.toUri + private val connectionTimeout = 5 seconds private val readTimeout = 1 minute - private val uri: URI = path.toUri private val responseBox: Box[HttpResponse[Array[Byte]]] = tryo( - path.getBasicAuthCredentials.map { credentials => + path.getBasicAuthCredential.map { credential => Http(uri.toString) .timeout(connTimeoutMs = connectionTimeout.toMillis.toInt, readTimeoutMs = readTimeout.toMillis.toInt) - .auth(credentials.user, credentials.password) + .auth(credential.user, credential.password) .asBytes }.getOrElse(Http(uri.toString) .timeout(connTimeoutMs = connectionTimeout.toMillis.toInt, readTimeoutMs = readTimeout.toMillis.toInt) diff --git a/webknossos-datastore/conf/META-INF/services/java.nio.file.spi.FileSystemProvider b/webknossos-datastore/conf/META-INF/services/java.nio.file.spi.FileSystemProvider deleted file mode 100644 index 6739b09b3c5..00000000000 --- a/webknossos-datastore/conf/META-INF/services/java.nio.file.spi.FileSystemProvider +++ /dev/null @@ -1,3 +0,0 @@ -com.scalableminds.webknossos.datastore.s3fs.S3FileSystemProvider -com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpsFileSystemProvider -com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpFileSystemProvider