Route to import explored datasources automatically (#7176)

* WIP: Option to import explored datasources automatically

* wip: enable autoAdd in explore route

* pass token

* assert new name, add error message

* remove unused imports

* move this feature to separate route

* undo frontend change

* changelog

* remove unused var

* unused import

* WIP: target folder

* folder path

* forbid forward slash when creating new folders

* logging

* Update Application.scala

* add evolution to forbid forward slashes in folder names

* fix after merge

* assert valid folder name in scala also in update route

CHANGELOG.unreleased.md

@@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Added
 - Added the option to change the ordering of color layers via drag and drop. This is useful when using the cover blend mode. [#7188](https://github.com/scalableminds/webknossos/pull/7188)
 - Added configuration to require users' emails to be verified, added flow to verify emails via link. [#7161](https://github.com/scalableminds/webknossos/pull/7161)
+- Added a route to explore and add remote datasets via API. [#7176](https://github.com/scalableminds/webknossos/pull/7176)
 
 ### Changed
 - Small messages during annotating (e.g. “finished undo”, “applying mapping…”) are now click-through so they do not block users from selecting tools. [7239](https://github.com/scalableminds/webknossos/pull/7239)

MIGRATIONS.unreleased.md

@@ -16,3 +16,4 @@ UPDATE webknossos.multiUsers SET isEmailVerified = false;
 
 ### Postgres Evolutions:
 - [105-verify-email.sql](conf/evolutions/105-verify-email.sql)
+- [106-folder-no-slashes.sql](conf/evolutions/106-folder-no-slashes.sql)

app/controllers/AuthenticationController.scala

@@ -577,9 +577,7 @@ class AuthenticationController @Inject()(
                   ) ?~> "user.creation.failed"
                   _ = analyticsService.track(SignupEvent(user, hadInvite = false))
                   multiUser <- multiUserDAO.findOne(user._multiUser)
-                  dataStoreToken <- bearerTokenAuthenticatorService
-                    .createAndInit(user.loginInfo, TokenType.DataStore, deleteOld = false)
-                    .toFox
+                  dataStoreToken <- bearerTokenAuthenticatorService.createAndInitDataStoreTokenForUser(user)
                   _ <- organizationService
                     .createOrganizationFolder(organization.name, dataStoreToken) ?~> "organization.folderCreation.failed"
                 } yield {

app/controllers/DataSetController.scala

@@ -9,7 +9,11 @@ import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, Elem
 import io.swagger.annotations._
 import models.analytics.{AnalyticsService, ChangeDatasetSettingsEvent, OpenDatasetEvent}
 import models.binary._
-import models.binary.explore.{ExploreRemoteDatasetParameters, ExploreRemoteLayerService}
+import models.binary.explore.{
+  ExploreAndAddRemoteDatasetParameters,
+  ExploreRemoteDatasetParameters,
+  ExploreRemoteLayerService
+}
 import models.organization.OrganizationDAO
 import models.team.{TeamDAO, TeamService}
 import models.user.{User, UserDAO, UserService}
@@ -25,6 +29,8 @@ import utils.{ObjectId, WkConf}
 import javax.inject.Inject
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.{ExecutionContext, Future}
+import com.scalableminds.util.tools.TristateOptionJsonHelper
+import models.folder.FolderService
 
 case class DatasetUpdateParameters(
     description: Option[Option[String]] = Some(None),
@@ -60,6 +66,7 @@ class DataSetController @Inject()(userService: UserService,
                                   wKRemoteSegmentAnythingClient: WKRemoteSegmentAnythingClient,
                                   teamService: TeamService,
                                   dataSetDAO: DataSetDAO,
+                                  folderService: FolderService,
                                   thumbnailService: ThumbnailService,
                                   thumbnailCachingService: ThumbnailCachingService,
                                   conf: WkConf,
@@ -125,6 +132,25 @@ class DataSetController @Inject()(userService: UserService,
       } yield Ok(Json.obj("dataSource" -> Json.toJson(dataSourceOpt), "report" -> reportMutable.mkString("\n")))
     }
 
+  @ApiOperation(hidden = true, value = "")
+  def exploreAndAddRemoteDataset(): Action[ExploreAndAddRemoteDatasetParameters] =
+    sil.SecuredAction.async(validateJson[ExploreAndAddRemoteDatasetParameters]) { implicit request =>
+      val reportMutable = ListBuffer[String]()
+      val adaptedParameters = ExploreRemoteDatasetParameters(request.body.remoteUri, None, None)
+      for {
+        dataSource <- exploreRemoteLayerService.exploreRemoteDatasource(List(adaptedParameters),
+                                                                        request.identity,
+                                                                        reportMutable)
+        _ <- bool2Fox(dataSource.dataLayers.nonEmpty) ?~> "explore.zeroLayers"
+        folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
+          folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "explore.autoAdd.getFolder.failed"
+        _ <- exploreRemoteLayerService.addRemoteDatasource(dataSource,
+                                                           request.body.datasetName,
+                                                           request.identity,
+                                                           folderIdOpt) ?~> "explore.autoAdd.failed"
+      } yield Ok
+    }
+
   @ApiOperation(value = "List all accessible datasets.", nickname = "datasetList")
   @ApiResponses(
     Array(new ApiResponse(code = 200, message = "JSON list containing one object per resulting dataset."),

app/controllers/FolderController.scala

@@ -57,6 +57,7 @@ class FolderController @Inject()(
         organization <- organizationDAO.findOne(request.identity._organization)
         _ <- folderDAO.findOne(idValidated) ?~> "folder.notFound"
         - <- Fox.assertTrue(folderDAO.isEditable(idValidated)) ?~> "folder.update.notAllowed" ~> FORBIDDEN
+        _ <- folderService.assertValidFolderName(params.name)
         _ <- folderDAO.updateName(idValidated, params.name) ?~> "folder.update.name.failed"
         _ <- folderService
           .updateAllowedTeams(idValidated, params.allowedTeams, request.identity) ?~> "folder.update.teams.failed"
@@ -115,6 +116,7 @@ class FolderController @Inject()(
   def create(parentId: String, name: String): Action[AnyContent] = sil.SecuredAction.async { implicit request =>
     for {
       parentIdValidated <- ObjectId.fromString(parentId)
+      _ <- folderService.assertValidFolderName(name)
       newFolder = Folder(ObjectId.generate, name)
       _ <- folderDAO.findOne(parentIdValidated) ?~> "folder.notFound"
       _ <- folderDAO.insertAsChild(parentIdValidated, newFolder) ?~> "folder.create.failed"

app/controllers/UserTokenController.scala

@@ -60,7 +60,7 @@ class UserTokenController @Inject()(dataSetDAO: DataSetDAO,
   def generateTokenForDataStore: Action[AnyContent] = sil.UserAwareAction.async { implicit request =>
     val tokenFox: Fox[String] = request.identity match {
       case Some(user) =>
-        bearerTokenService.createAndInit(user.loginInfo, TokenType.DataStore, deleteOld = false).toFox
+        bearerTokenService.createAndInitDataStoreTokenForUser(user)
       case None => Fox.successful("")
     }
     for {

app/models/binary/DataStore.scala

@@ -130,6 +130,13 @@ class DataStoreDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext
       parsed <- parseAll(r)
     } yield parsed
 
+  def findOneWithUploadsAllowed(implicit ctx: DBAccessContext): Fox[DataStore] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(q"select $columns from $existingCollectionName where allowsUpload and $accessQuery".as[DatastoresRow])
+      parsed <- parseFirst(r, "find one with uploads allowed")
+    } yield parsed
+
   def updateUrlByName(name: String, url: String): Fox[Unit] = {
     val query = for { row <- Datastores if notdel(row) && row.name === name } yield row.url
     for { _ <- run(query.update(url)) } yield ()

app/models/binary/WKRemoteDataStoreClient.scala

@@ -2,12 +2,14 @@ package models.binary
 
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Int}
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, GenericDataSource}
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.services.DirectoryStorageReport
 import com.typesafe.scalalogging.LazyLogging
 import controllers.RpcTokenHolder
 import play.api.libs.json.JsObject
 import play.utils.UriEncoding
+import utils.ObjectId
 
 class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLogging {
 
@@ -69,4 +71,16 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
       .silent
       .getWithJsonResponse[List[DirectoryStorageReport]]
 
+  def addDataSource(organizationName: String,
+                    datasetName: String,
+                    dataSource: GenericDataSource[DataLayer],
+                    folderId: Option[ObjectId],
+                    userToken: String): Fox[Unit] =
+    for {
+      _ <- rpc(s"${dataStore.url}/data/datasets/$organizationName/$datasetName")
+        .addQueryString("token" -> userToken)
+        .addQueryStringOptional("folderId", folderId.map(_.toString))
+        .put(dataSource)
+    } yield ()
+
 }

app/models/binary/explore/ExploreRemoteLayerService.scala

@@ -1,5 +1,6 @@
 package models.binary.explore
 
+import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.geometry.{Vec3Double, Vec3Int}
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.dataformats.n5.{N5DataLayer, N5SegmentationLayer}
@@ -15,14 +16,18 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr._
 import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3ArrayHeader
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource._
+import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor}
 import com.typesafe.scalalogging.LazyLogging
+import models.binary.{DataSetService, DataStoreDAO, WKRemoteDataStoreClient}
 import models.binary.credential.CredentialService
+import models.organization.OrganizationDAO
 import models.user.User
 import net.liftweb.common.{Empty, Failure, Full}
 import net.liftweb.util.Helpers.tryo
-import oxalis.security.WkEnv
+import oxalis.security.{WkEnv, WkSilhouetteEnvironment}
 import play.api.libs.json.{Json, OFormat}
+import utils.ObjectId
 
 import java.net.URI
 import javax.inject.Inject
@@ -38,10 +43,25 @@ object ExploreRemoteDatasetParameters {
   implicit val jsonFormat: OFormat[ExploreRemoteDatasetParameters] = Json.format[ExploreRemoteDatasetParameters]
 }
 
-class ExploreRemoteLayerService @Inject()(credentialService: CredentialService, dataVaultService: DataVaultService)
+case class ExploreAndAddRemoteDatasetParameters(remoteUri: String, datasetName: String, folderPath: Option[String])
+
+object ExploreAndAddRemoteDatasetParameters {
+  implicit val jsonFormat: OFormat[ExploreAndAddRemoteDatasetParameters] =
+    Json.format[ExploreAndAddRemoteDatasetParameters]
+}
+
+class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
+                                          dataVaultService: DataVaultService,
+                                          organizationDAO: OrganizationDAO,
+                                          dataStoreDAO: DataStoreDAO,
+                                          dataSetService: DataSetService,
+                                          wkSilhouetteEnvironment: WkSilhouetteEnvironment,
+                                          rpc: RPC)
     extends FoxImplicits
     with LazyLogging {
 
+  private lazy val bearerTokenService = wkSilhouetteEnvironment.combinedAuthenticatorService.tokenAuthenticatorService
+
   def exploreRemoteDatasource(
       urisWithCredentials: List[ExploreRemoteDatasetParameters],
       requestIdentity: WkEnv#I,
@@ -67,6 +87,20 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       )
     } yield dataSource
 
+  def addRemoteDatasource(dataSource: GenericDataSource[DataLayer],
+                          datasetName: String,
+                          user: User,
+                          folderId: Option[ObjectId])(implicit ctx: DBAccessContext): Fox[Unit] =
+    for {
+      organization <- organizationDAO.findOne(user._organization)
+      dataStore <- dataStoreDAO.findOneWithUploadsAllowed
+      _ <- dataSetService.assertValidDataSetName(datasetName)
+      _ <- dataSetService.assertNewDataSetName(datasetName, organization._id) ?~> "dataSet.name.alreadyTaken"
+      client = new WKRemoteDataStoreClient(dataStore, rpc)
+      userToken <- bearerTokenService.createAndInitDataStoreTokenForUser(user)
+      _ <- client.addDataSource(organization.name, datasetName, dataSource, folderId, userToken)
+    } yield ()
+
   private def makeLayerNamesUnique(layers: List[DataLayer]): List[DataLayer] = {
     val namesSetMutable = scala.collection.mutable.Set[String]()
     layers.map { layer: DataLayer =>

app/models/folder/Folder.scala

@@ -2,7 +2,9 @@ package models.folder
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.util.tools.Fox.{bool2Fox, option2Fox}
 import com.scalableminds.webknossos.schema.Tables._
+import com.typesafe.scalalogging.LazyLogging
 import models.organization.{Organization, OrganizationDAO}
 import models.team.{TeamDAO, TeamService}
 import models.user.User
@@ -14,6 +16,7 @@ import utils.sql.{SQLDAO, SqlClient, SqlToken}
 import utils.ObjectId
 
 import javax.inject.Inject
+import scala.annotation.tailrec
 import scala.concurrent.ExecutionContext
 
 case class Folder(_id: ObjectId, name: String)
@@ -28,7 +31,8 @@ object FolderParameters {
 class FolderService @Inject()(teamDAO: TeamDAO,
                               teamService: TeamService,
                               folderDAO: FolderDAO,
-                              organizationDAO: OrganizationDAO)(implicit ec: ExecutionContext) {
+                              organizationDAO: OrganizationDAO)(implicit ec: ExecutionContext)
+    extends LazyLogging {
 
   val defaultRootName: String = "Datasets"
 
@@ -71,6 +75,54 @@ class FolderService @Inject()(teamDAO: TeamDAO,
       _ <- teamDAO.updateAllowedTeamsForFolder(folderId, newTeamIds)
     } yield ()
 
+  def assertValidFolderName(name: String): Fox[Unit] =
+    bool2Fox(!name.contains("/")) ?~> "folder.nameMustNotContainSlash"
+
+  def getOrCreateFromPathLiteral(folderPathLiteral: String, organizationId: ObjectId)(
+      implicit ctx: DBAccessContext): Fox[ObjectId] =
+    for {
+      organization <- organizationDAO.findOne(organizationId)
+      foldersWithParents: Seq[FolderWithParent] <- folderDAO.findTreeOf(organization._rootFolder)
+      root <- foldersWithParents.find(_._parent.isEmpty).toFox
+      _ <- bool2Fox(folderPathLiteral.startsWith("/")) ?~> "pathLiteral.mustStartWithSlash"
+      pathNames = folderPathLiteral.drop(1).split("/").toList
+      suppliedRootName <- pathNames.headOption.toFox
+      _ <- bool2Fox(suppliedRootName == root.name) ?~> "pathLiteral.mustStartAtOrganizationRootFolder"
+      (existingFolderId, remainingPathNames) = findLowestMatchingFolder(root, foldersWithParents, pathNames)
+      _ = if (remainingPathNames.nonEmpty) {
+        logger.info(s"Creating new folder(s) under $existingFolderId by path literal: $remainingPathNames...")
+      }
+      targetFolderId <- createMissingFoldersForPathNames(existingFolderId, remainingPathNames)
+    } yield targetFolderId
+
+  private def findLowestMatchingFolder(root: FolderWithParent,
+                                       foldersWithParents: Seq[FolderWithParent],
+                                       pathNames: List[String]): (ObjectId, List[String]) = {
+
+    @tailrec
+    def findFolderIter(currentParent: FolderWithParent, remainingPathNames: List[String]): (ObjectId, List[String]) = {
+      val nextOpt = foldersWithParents.find(folder =>
+        folder._parent.contains(currentParent._id) && remainingPathNames.headOption.contains(folder.name))
+      nextOpt match {
+        case Some(next) => findFolderIter(next, remainingPathNames.drop(1))
+        case None       => (currentParent._id, remainingPathNames)
+      }
+    }
+
+    findFolderIter(root, pathNames.drop(1))
+  }
+
+  private def createMissingFoldersForPathNames(parentFolderId: ObjectId, remainingPathNames: List[String])(
+      implicit ctx: DBAccessContext): Fox[ObjectId] =
+    remainingPathNames match {
+      case pathNamesHead :: pathNamesTail =>
+        for {
+          newFolder <- Fox.successful(Folder(ObjectId.generate, pathNamesHead))
+          _ <- folderDAO.insertAsChild(parentFolderId, newFolder)
+          folderId <- createMissingFoldersForPathNames(newFolder._id, pathNamesTail)
+        } yield folderId
+      case Nil => Fox.successful(parentFolderId)
+    }
 }
 
 class FolderDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)

app/oxalis/security/WebknossosBearerTokenAuthenticatorService.scala

@@ -61,6 +61,9 @@ class WebknossosBearerTokenAuthenticatorService(settings: BearerTokenAuthenticat
         case e => throw new AuthenticatorInitializationException(InitError.format(ID, authenticator), e)
       }
 
+  def createAndInitDataStoreTokenForUser(user: User): Fox[String] =
+    createAndInit(user.loginInfo, TokenType.DataStore, deleteOld = false)
+
   def createAndInit(loginInfo: LoginInfo, tokenType: TokenType, deleteOld: Boolean = true): Future[String] =
     for {
       tokenAuthenticator <- create(loginInfo, tokenType)

conf/evolutions/106-folder-no-slashes.sql

@@ -0,0 +1,15 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 105, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+DROP VIEW webknossos.folders_;
+
+UPDATE webknossos.folders SET name = REPLACE(name, '/', '_') WHERE name ~ '/';
+
+ALTER TABLE webknossos.folders ADD CONSTRAINT folders_name_check CHECK (name !~ '/');
+
+CREATE VIEW webknossos.folders_ as SELECT * FROM webknossos.folders WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 106;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/106-folder-no-slashes.sql

@@ -0,0 +1,15 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 106, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+DROP VIEW webknossos.folders_;
+
+-- set cannot be undone (we don’t want to turn all underscores into slashes)
+
+ALTER TABLE webknossos.folders DROP CONSTRAINT folders_name_check;
+
+CREATE VIEW webknossos.folders_ as SELECT * FROM webknossos.folders WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 105;
+
+COMMIT TRANSACTION;

conf/messages

@@ -110,6 +110,7 @@ dataSet.upload.noFiles=Tried to finish upload with no files. May be a retry of a
 dataSet.upload.storageExceeded=Cannot upload dataset because the storage quota of the organization is exceeded.
 dataSet.explore.failed.readFile=Failed to read remote file
 dataSet.explore.magDtypeMismatch=Element class must be the same for all mags of a layer. Got {0}
+dataSet.explore.autoAdd.failed=Failed to automatically import the explored dataset.
 
 dataVault.insert.failed=Failed to store remote file system credential
 dataVault.setup.failed=Failed to set up remote file system
@@ -334,6 +335,7 @@ folder.update.name.failed=Failed to update the folder’s name
 folder.update.teams.failed=Failed to update the folder’s allowed teams
 folder.create.failed.teams.failed=Failed to create folder in this location
 folder.noWriteAccess=No write access in this folder
+folder.nameMustNotContainSlash=Folder names cannot contain forward slashes
 
 segmentAnything.notEnabled=AI based quick select is not enabled for this WEBKNOSSOS instance.
 segmentAnything.noUri=No Uri for SAM server configured.