Enable publications to link to annotations

CHANGELOG.unreleased.md

@@ -15,6 +15,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - WebKnossos now remembers the tool that was active in between disabling and enabling the segmentation layer. [#6362](https://github.com/scalableminds/webknossos/pull/6362)
 - Segmentation layers which were not previously editable now show an (un)lock icon button which shortcuts to the Add Volume Layer modal with the layer being preselected. [#6330](https://github.com/scalableminds/webknossos/pull/6330)
 - The NML file in volume annotation download now includes segment metadata like names and anchor positions. [#6347](https://github.com/scalableminds/webknossos/pull/6347)
+- Added new backend API route for requesting all publications. Those publications can now have also attached annotations. [#6315](https://github.com/scalableminds/webknossos/pull/6315)
+
 
 ### Changed
 - The sharing modal now automatically saves changes of the sharing options. [#6314](https://github.com/scalableminds/webknossos/pull/6314)

MIGRATIONS.unreleased.md

@@ -10,3 +10,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 ### Postgres Evolutions:
 - [084-annotation-contributors.sql](conf/evolutions/084-annotation-contributors.sql)
+- [085-add-annotations-publicationforeignkey](conf/evolutions/085-add-annotations-publicationforeignkey.sql)

app/controllers/AnnotationController.scala

@@ -27,6 +27,12 @@ import play.api.libs.json.{JsArray, _}
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import utils.{ObjectId, WkConf}
 
+import javax.inject.Inject
+import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent}
+import models.annotation.AnnotationLayerType.AnnotationLayerType
+import models.organization.OrganizationDAO
+import oxalis.mail.{MailchimpClient, MailchimpTag}
+
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 

app/controllers/DataSetController.scala

@@ -184,8 +184,8 @@ class DataSetController @Inject()(userService: UserService,
                 dataSetService.publicWrites(
                   d,
                   requestingUser,
-                  organization,
-                  dataStore,
+                  Some(organization),
+                  Some(dataStore),
                   skipResolutions = true,
                   requestingUserTeamManagerMemberships) ?~> Messages("dataset.list.writesFailed", d.name)
               }
@@ -233,7 +233,7 @@ class DataSetController @Inject()(userService: UserService,
             dataSetLastUsedTimesDAO.updateForDataSetAndUser(dataSet._id, user._id))
           // Access checked above via dataset. In case of shared dataset/annotation, show datastore even if not otherwise accessible
           dataStore <- dataSetService.dataStoreFor(dataSet)(GlobalAccessContext)
-          js <- dataSetService.publicWrites(dataSet, request.identity, organization, dataStore)
+          js <- dataSetService.publicWrites(dataSet, request.identity, Some(organization), Some(dataStore))
           _ = request.identity.map { user =>
             analyticsService.track(OpenDatasetEvent(user, dataSet))
             if (dataSet.isPublic) {
@@ -307,7 +307,7 @@ Expects:
             _ = analyticsService.track(ChangeDatasetSettingsEvent(request.identity, updated))
             organization <- organizationDAO.findOne(updated._organization)(GlobalAccessContext)
             dataStore <- dataSetService.dataStoreFor(updated)
-            js <- dataSetService.publicWrites(updated, Some(request.identity), organization, dataStore)
+            js <- dataSetService.publicWrites(updated, Some(request.identity), Some(organization), Some(dataStore))
           } yield Ok(Json.toJson(js))
       }
     }

app/controllers/PublicationController.scala

@@ -0,0 +1,47 @@
+package controllers
+
+import com.mohiva.play.silhouette.api.Silhouette
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
+import io.swagger.annotations._
+import models.annotation.AnnotationDAO
+import models.binary.{DataSetDAO, PublicationDAO, PublicationService}
+import oxalis.security.WkEnv
+import play.api.libs.json._
+import play.api.mvc.{Action, AnyContent}
+import utils.ObjectId
+
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+@Api
+class PublicationController @Inject()(publicationService: PublicationService,
+                                      publicationDAO: PublicationDAO,
+                                      dataSetDAO: DataSetDAO,
+                                      annotationDAO: AnnotationDAO,
+                                      sil: Silhouette[WkEnv])(implicit ec: ExecutionContext)
+    extends Controller
+    with ProtoGeometryImplicits
+    with FoxImplicits {
+
+  @ApiOperation(value = "Information about a publication", nickname = "publicationInfo")
+  @ApiResponses(
+    Array(new ApiResponse(code = 200, message = "JSON object containing information about this publication."),
+          new ApiResponse(code = 400, message = badRequestLabel)))
+  def read(@ApiParam(value = "The id of the publication") publicationId: String): Action[AnyContent] =
+    sil.UserAwareAction.async { implicit request =>
+      for {
+        publication <- publicationDAO.findOne(ObjectId(publicationId)) ?~> "publication.notFound" ~> NOT_FOUND
+        js <- publicationService.publicWritesWithDatasetsAndAnnotations(publication)
+      } yield Ok(js)
+    }
+
+  def listPublications: Action[AnyContent] = sil.UserAwareAction.async { implicit request =>
+    {
+      for {
+        publications <- publicationDAO.findAll ?~> "publication.notFound" ~> NOT_FOUND
+        jsResult <- Fox.serialCombined(publications)(publicationService.publicWritesWithDatasetsAndAnnotations)
+      } yield Ok(Json.toJson(jsResult))
+    }
+  }
+}

app/models/annotation/Annotation.scala

@@ -344,6 +344,15 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
       parsed <- parseAll(r)
     } yield parsed
 
+  def findAllByPublication(publicationId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[Annotation]] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(
+        sql"select #$columns from #$existingCollectionName where _publication = $publicationId and #$accessQuery"
+          .as[AnnotationsRow]).map(_.toList)
+      parsed <- parseAll(r)
+    } yield parsed
+
   def findOneByTracingId(tracingId: String)(implicit ctx: DBAccessContext): Fox[Annotation] =
     for {
       annotationId <- annotationLayerDAO.findAnnotationIdByTracingId(tracingId)

app/models/annotation/AnnotationService.scala

@@ -1,7 +1,6 @@
 package models.annotation
 
 import java.io.{BufferedOutputStream, File, FileOutputStream}
-
 import akka.actor.ActorSystem
 import akka.stream.Materializer
 import com.scalableminds.util.accesscontext.{AuthorizedAccessContext, DBAccessContext, GlobalAccessContext}
@@ -31,6 +30,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{
 }
 import com.typesafe.scalalogging.LazyLogging
 import controllers.AnnotationLayerParameters
+
 import javax.inject.Inject
 import models.annotation.AnnotationState._
 import models.annotation.AnnotationType.AnnotationType
@@ -98,7 +98,8 @@ class AnnotationService @Inject()(
     temporaryFileCreator: TemporaryFileCreator,
     meshDAO: MeshDAO,
     meshService: MeshService,
-    sharedAnnotationsDAO: SharedAnnotationsDAO)(implicit ec: ExecutionContext, val materializer: Materializer)
+    sharedAnnotationsDAO: SharedAnnotationsDAO
+)(implicit ec: ExecutionContext, val materializer: Materializer)
     extends BoxImplicits
     with FoxImplicits
     with ProtoGeometryImplicits
@@ -823,6 +824,20 @@ class AnnotationService @Inject()(
     }
   }
 
+  def writesWithDataset(annotation: Annotation): Fox[JsObject] = {
+    implicit val ctx: DBAccessContext = GlobalAccessContext
+    for {
+      dataSet <- dataSetDAO.findOne(annotation._dataSet) ?~> "dataSet.notFoundForAnnotation"
+      tracingStore <- tracingStoreDAO.findFirst
+      tracingStoreJs <- tracingStoreService.publicWrites(tracingStore)
+      dataSetJs <- dataSetService.publicWrites(dataSet, None, None, None)
+    } yield
+      Json.obj("id" -> annotation._id,
+               "typ" -> annotation.typ,
+               "tracingStore" -> tracingStoreJs,
+               "dataSet" -> dataSetJs)
+  }
+
   private def userJsonForAnnotation(userId: ObjectId, userOpt: Option[User] = None): Fox[Option[JsObject]] =
     if (userId == ObjectId.dummyId) {
       Fox.successful(None)

app/models/binary/DataSet.scala

@@ -175,6 +175,15 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
       parsed <- parseAll(r)
     } yield parsed
 
+  def findAllByPublication(publicationId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[DataSet]] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(
+        sql"select #$columns from #$existingCollectionName where _publication = $publicationId and #$accessQuery"
+          .as[DatasetsRow]).map(_.toList)
+      parsed <- parseAll(r)
+    } yield parsed
+
   /* Disambiguation method for legacy URLs and NMLs: if the user has access to multiple datasets of the same name, use the oldest.
    * This is reasonable, because the legacy URL/NML was likely created before this ambiguity became possible */
   def getOrganizationForDataSet(dataSetName: String)(implicit ctx: DBAccessContext): Fox[ObjectId] =

app/models/binary/DataSetService.scala

@@ -37,7 +37,7 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
                                teamDAO: TeamDAO,
                                workerDAO: WorkerDAO,
                                publicationDAO: PublicationDAO,
-                               publicationService: PublicationService,
+                               publicationService: CompactPublicationService,
                                dataStoreService: DataStoreService,
                                teamService: TeamService,
                                userService: UserService,
@@ -343,12 +343,18 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
 
   def publicWrites(dataSet: DataSet,
                    requestingUserOpt: Option[User],
-                   organization: Organization,
-                   dataStore: DataStore,
+                   organization: Option[Organization],
+                   dataStore: Option[DataStore],
                    skipResolutions: Boolean = false,
                    requestingUserTeamManagerMemberships: Option[List[TeamMembership]] = None)(
       implicit ctx: DBAccessContext): Fox[JsObject] =
     for {
+      organization <- Fox.fillOption(organization) {
+        organizationDAO.findOne(dataSet._organization) ?~> "organization.notFound"
+      }
+      dataStore <- Fox.fillOption(dataStore) {
+        dataStoreFor(dataSet)
+      }
       teams <- allowedTeamsFor(dataSet._id, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed"
       teamsJs <- Fox.serialCombined(teams)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed"
       logoUrl <- logoUrlFor(dataSet, Some(organization)) ?~> "dataset.list.fetchLogoUrlFailed"

app/models/binary/Publication.scala

@@ -1,8 +1,12 @@
 package models.binary
 
+import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.schema.Tables._
 import javax.inject.Inject
+import models.annotation.{AnnotationDAO, AnnotationService}
+import play.api.http.Status.NOT_FOUND
+import play.api.libs.json.Format.GenericFormat
 import play.api.libs.json.{JsObject, Json}
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.Rep
@@ -18,7 +22,8 @@ case class Publication(_id: ObjectId,
                        created: Long = System.currentTimeMillis(),
                        isDeleted: Boolean = false)
 
-class PublicationService @Inject()()(implicit ec: ExecutionContext) {
+class CompactPublicationService @Inject()()(implicit ec: ExecutionContext) {
+  // TODO Does the dataset need to know the publication?
   def publicWrites(p: Publication): Fox[JsObject] =
     Fox.successful(
       Json.obj(
@@ -31,6 +36,34 @@ class PublicationService @Inject()()(implicit ec: ExecutionContext) {
       ))
 }
 
+class PublicationService @Inject()(dataSetService: DataSetService,
+                                   dataSetDAO: DataSetDAO,
+                                   annotationService: AnnotationService,
+                                   annotationDAO: AnnotationDAO)(implicit ec: ExecutionContext) {
+
+  def publicWritesWithDatasetsAndAnnotations(publication: Publication): Fox[JsObject] = {
+    implicit val ctx: DBAccessContext = GlobalAccessContext
+    for {
+      dataSets <- dataSetDAO.findAllByPublication(publication._id) ?~> "not found" ~> NOT_FOUND
+      annotations <- annotationDAO.findAllByPublication(publication._id) ?~> "not found" ~> NOT_FOUND
+      dataSetsJson <- Fox.serialCombined(dataSets)(d => dataSetService.publicWrites(d, None, None, None))
+      annotationsJson <- Fox.serialCombined(annotations) { annotation =>
+        annotationService.writesWithDataset(annotation)
+      }
+    } yield
+      Json.obj(
+        "id" -> publication._id.id,
+        "publicationDate" -> publication.publicationDate,
+        "imageUrl" -> publication.imageUrl,
+        "title" -> publication.title,
+        "description" -> publication.description,
+        "created" -> publication.created,
+        "datasets" -> dataSetsJson,
+        "annotations" -> annotationsJson
+      )
+  }
+}
+
 class PublicationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext)
     extends SQLDAO[Publication, PublicationsRow, Publications](sqlClient) {
   val collection = Publications
@@ -52,6 +85,18 @@ class PublicationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionConte
       )
     )
 
+  override def findOne(id: ObjectId)(implicit ctx: DBAccessContext): Fox[Publication] =
+    for {
+      r <- run(sql"select #$columns from #$existingCollectionName where _id = ${id.id}".as[PublicationsRow])
+      parsed <- parseFirst(r, id)
+    } yield parsed
+
+  override def findAll(implicit ctx: DBAccessContext): Fox[List[Publication]] =
+    for {
+      r <- run(sql"select #$columns from #$existingCollectionName".as[PublicationsRow])
+      parsed <- parseAll(r)
+    } yield parsed
+
   def insertOne(p: Publication): Fox[Unit] =
     for {
       _ <- run(

conf/evolutions/085-annotation-publication.sql

@@ -0,0 +1,12 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+ALTER TABLE webknossos.annotations
+  ADD COLUMN _publication CHAR(24),
+  ADD CONSTRAINT publication_ref FOREIGN KEY(_publication) REFERENCES webknossos.publications(_id) DEFERRABLE;
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 85;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/085-annotation-publication.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+ALTER TABLE webknossos.annotations
+  DROP COLUMN _publication CHAR(24);
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 84;
+
+COMMIT TRANSACTION;

conf/messages

@@ -363,6 +363,8 @@ job.meshFile.notAllowed.organization = Calculating mesh files is only allowed fo
 job.globalizeFloodfill.notAllowed.organization = Globalizing floodfills is only allowed for datasets of your own organization.
 job.applyMergerMode.notAllowed.organization = Applying merger mode tracings is only allowed for datasets of your own organization.
 
+publication.notFound = Publication could not be found.
+
 agglomerateSkeleton.failed=Could not generate agglomerate skeleton.
 
 unexpectedReturn=Unexpected return of parameters

conf/webknossos.latest.routes

@@ -223,3 +223,7 @@ POST          /jobs/run/materializeVolumeAnnotation/:organizationName/:dataSetNa
 GET           /jobs/:id                                                                 controllers.JobsController.get(id: String)
 PATCH         /jobs/:id/cancel                                                          controllers.JobsController.cancel(id: String)
 POST          /jobs/:id/status                                                          controllers.WKRemoteWorkerController.updateJobStatus(key: String, id: String)
+
+# Publications
+GET           /publications                                                             controllers.PublicationController.listPublications()
+GET           /publications/:id                                                         controllers.PublicationController.read(id: String)