Enable publications to link to annotations (#6315)

* add schema change, publicationDAO * add one more step * add backend routes for publication * fix csv file * add more * update db queries * reformat * remove unnecessary parts * remove unnecessary parts * bump schema.sql version to 84 * remove unneeded import * add Changelog entry * incroporate feedback * update evolution * use comma instead of semicolon * remove unused variables * update schema version * adds frontend for new publication API * lint * fixes * rename /publication/:id -> /publications/:id * remove publication from datasets * e2e * rm CompactPublicationService * pr feedback * pr feedback * snapshots * remove unused stuff Co-authored-by: Florian M <[email protected]> Co-authored-by: Norman Rzepka <[email protected]> Co-authored-by: Florian M <[email protected]>
scalableminds · Aug 4, 2022 · ae0b8b5 · ae0b8b5
1 parent b93e00e
commit ae0b8b5
Show file tree

Hide file tree

Showing 27 changed files with 402 additions and 254 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -15,6 +15,8 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - WebKnossos now remembers the tool that was active in between disabling and enabling the segmentation layer. [#6362](https://github.com/scalableminds/webknossos/pull/6362)
 - Segmentation layers which were not previously editable now show an (un)lock icon button which shortcuts to the Add Volume Layer modal with the layer being preselected. [#6330](https://github.com/scalableminds/webknossos/pull/6330)
 - The NML file in volume annotation download now includes segment metadata like names and anchor positions. [#6347](https://github.com/scalableminds/webknossos/pull/6347)
+- Added new backend API route for requesting all publications. Those publications can now have also attached annotations. [#6315](https://github.com/scalableminds/webknossos/pull/6315)
+
 
 ### Changed
 - webKnossos uses WebGL 2 instead of WebGL 1 now. In case your browser/hardware does not support this, webKnossos will alert you and you need to upgrade your system. [#6350](https://github.com/scalableminds/webknossos/pull/6350)

diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md
@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 
 ### Postgres Evolutions:
 - [084-annotation-contributors.sql](conf/evolutions/084-annotation-contributors.sql)
+- [085-add-annotations-publicationforeignkey](conf/evolutions/085-add-annotations-publicationforeignkey.sql)
diff --git a/app/controllers/AnnotationController.scala b/app/controllers/AnnotationController.scala
@@ -8,25 +8,26 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.tracingstore.tracings.volume.ResolutionRestrictions
 import com.scalableminds.webknossos.tracingstore.tracings.{TracingIds, TracingType}
 import io.swagger.annotations._
-import javax.inject.Inject
-import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent}
-import models.annotation.AnnotationLayerType.AnnotationLayerType
 import models.annotation.AnnotationState.Cancelled
 import models.annotation._
 import models.binary.{DataSetDAO, DataSetService}
-import models.organization.OrganizationDAO
 import models.project.ProjectDAO
 import models.task.TaskDAO
 import models.team.{TeamDAO, TeamService}
 import models.user.time._
 import models.user.{User, UserDAO, UserService}
-import oxalis.mail.{MailchimpClient, MailchimpTag}
 import oxalis.security.{URLSharing, WkEnv}
 import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.json.{JsArray, _}
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import utils.{ObjectId, WkConf}
 
+import javax.inject.Inject
+import models.analytics.{AnalyticsService, CreateAnnotationEvent, OpenAnnotationEvent}
+import models.annotation.AnnotationLayerType.AnnotationLayerType
+import models.organization.OrganizationDAO
+import oxalis.mail.{MailchimpClient, MailchimpTag}
+
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 

diff --git a/app/controllers/DataSetController.scala b/app/controllers/DataSetController.scala
@@ -184,8 +184,8 @@ class DataSetController @Inject()(userService: UserService,
                 dataSetService.publicWrites(
                   d,
                   requestingUser,
-                  organization,
-                  dataStore,
+                  Some(organization),
+                  Some(dataStore),
                   skipResolutions = true,
                   requestingUserTeamManagerMemberships) ?~> Messages("dataset.list.writesFailed", d.name)
               }
@@ -233,7 +233,7 @@ class DataSetController @Inject()(userService: UserService,
             dataSetLastUsedTimesDAO.updateForDataSetAndUser(dataSet._id, user._id))
           // Access checked above via dataset. In case of shared dataset/annotation, show datastore even if not otherwise accessible
           dataStore <- dataSetService.dataStoreFor(dataSet)(GlobalAccessContext)
-          js <- dataSetService.publicWrites(dataSet, request.identity, organization, dataStore)
+          js <- dataSetService.publicWrites(dataSet, request.identity, Some(organization), Some(dataStore))
           _ = request.identity.map { user =>
             analyticsService.track(OpenDatasetEvent(user, dataSet))
             if (dataSet.isPublic) {
@@ -307,7 +307,7 @@ Expects:
             _ = analyticsService.track(ChangeDatasetSettingsEvent(request.identity, updated))
             organization <- organizationDAO.findOne(updated._organization)(GlobalAccessContext)
             dataStore <- dataSetService.dataStoreFor(updated)
-            js <- dataSetService.publicWrites(updated, Some(request.identity), organization, dataStore)
+            js <- dataSetService.publicWrites(updated, Some(request.identity), Some(organization), Some(dataStore))
           } yield Ok(Json.toJson(js))
       }
     }

diff --git a/app/controllers/PublicationController.scala b/app/controllers/PublicationController.scala
@@ -0,0 +1,44 @@
+package controllers
+
+import com.mohiva.play.silhouette.api.Silhouette
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.helpers.ProtoGeometryImplicits
+import io.swagger.annotations._
+import javax.inject.Inject
+import models.binary.{PublicationDAO, PublicationService}
+import oxalis.security.WkEnv
+import play.api.libs.json._
+import play.api.mvc.{Action, AnyContent}
+import utils.ObjectId
+
+import scala.concurrent.ExecutionContext
+
+@Api
+class PublicationController @Inject()(publicationService: PublicationService,
+                                      publicationDAO: PublicationDAO,
+                                      sil: Silhouette[WkEnv])(implicit ec: ExecutionContext)
+    extends Controller
+    with ProtoGeometryImplicits
+    with FoxImplicits {
+
+  @ApiOperation(value = "Information about a publication", nickname = "publicationInfo")
+  @ApiResponses(
+    Array(new ApiResponse(code = 200, message = "JSON object containing information about this publication."),
+          new ApiResponse(code = 400, message = badRequestLabel)))
+  def read(@ApiParam(value = "The id of the publication") publicationId: String): Action[AnyContent] =
+    sil.UserAwareAction.async { implicit request =>
+      for {
+        publication <- publicationDAO.findOne(ObjectId(publicationId)) ?~> "publication.notFound" ~> NOT_FOUND
+        js <- publicationService.publicWrites(publication)
+      } yield Ok(js)
+    }
+
+  def listPublications: Action[AnyContent] = sil.UserAwareAction.async { implicit request =>
+    {
+      for {
+        publications <- publicationDAO.findAll ?~> "publication.notFound" ~> NOT_FOUND
+        jsResult <- Fox.serialCombined(publications)(publicationService.publicWrites)
+      } yield Ok(Json.toJson(jsResult))
+    }
+  }
+}
diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala
@@ -344,6 +344,15 @@ class AnnotationDAO @Inject()(sqlClient: SQLClient, annotationLayerDAO: Annotati
       parsed <- parseAll(r)
     } yield parsed
 
+  def findAllByPublication(publicationId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[Annotation]] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(
+        sql"select #$columns from #$existingCollectionName where _publication = $publicationId and #$accessQuery"
+          .as[AnnotationsRow]).map(_.toList)
+      parsed <- parseAll(r)
+    } yield parsed
+
   def findOneByTracingId(tracingId: String)(implicit ctx: DBAccessContext): Fox[Annotation] =
     for {
       annotationId <- annotationLayerDAO.findAnnotationIdByTracingId(tracingId)

diff --git a/app/models/annotation/AnnotationService.scala b/app/models/annotation/AnnotationService.scala
@@ -1,7 +1,6 @@
 package models.annotation
 
 import java.io.{BufferedOutputStream, File, FileOutputStream}
-
 import akka.actor.ActorSystem
 import akka.stream.Materializer
 import com.scalableminds.util.accesscontext.{AuthorizedAccessContext, DBAccessContext, GlobalAccessContext}
@@ -31,6 +30,7 @@ import com.scalableminds.webknossos.tracingstore.tracings.volume.{
 }
 import com.typesafe.scalalogging.LazyLogging
 import controllers.AnnotationLayerParameters
+
 import javax.inject.Inject
 import models.annotation.AnnotationState._
 import models.annotation.AnnotationType.AnnotationType
@@ -98,7 +98,8 @@ class AnnotationService @Inject()(
     temporaryFileCreator: TemporaryFileCreator,
     meshDAO: MeshDAO,
     meshService: MeshService,
-    sharedAnnotationsDAO: SharedAnnotationsDAO)(implicit ec: ExecutionContext, val materializer: Materializer)
+    sharedAnnotationsDAO: SharedAnnotationsDAO
+)(implicit ec: ExecutionContext, val materializer: Materializer)
     extends BoxImplicits
     with FoxImplicits
     with ProtoGeometryImplicits
@@ -823,6 +824,24 @@ class AnnotationService @Inject()(
     }
   }
 
+  def writesWithDataset(annotation: Annotation): Fox[JsObject] = {
+    implicit val ctx: DBAccessContext = GlobalAccessContext
+    for {
+      dataSet <- dataSetDAO.findOne(annotation._dataSet) ?~> "dataSet.notFoundForAnnotation"
+      tracingStore <- tracingStoreDAO.findFirst
+      tracingStoreJs <- tracingStoreService.publicWrites(tracingStore)
+      dataSetJs <- dataSetService.publicWrites(dataSet, None, None, None)
+    } yield
+      Json.obj(
+        "id" -> annotation._id.id,
+        "name" -> annotation.name,
+        "description" -> annotation.description,
+        "typ" -> annotation.typ,
+        "tracingStore" -> tracingStoreJs,
+        "dataSet" -> dataSetJs
+      )
+  }
+
   private def userJsonForAnnotation(userId: ObjectId, userOpt: Option[User] = None): Fox[Option[JsObject]] =
     if (userId == ObjectId.dummyId) {
       Fox.successful(None)

diff --git a/app/models/binary/DataSet.scala b/app/models/binary/DataSet.scala
@@ -175,6 +175,15 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
       parsed <- parseAll(r)
     } yield parsed
 
+  def findAllByPublication(publicationId: ObjectId)(implicit ctx: DBAccessContext): Fox[List[DataSet]] =
+    for {
+      accessQuery <- readAccessQuery
+      r <- run(
+        sql"select #$columns from #$existingCollectionName where _publication = $publicationId and #$accessQuery"
+          .as[DatasetsRow]).map(_.toList)
+      parsed <- parseAll(r)
+    } yield parsed
+
   /* Disambiguation method for legacy URLs and NMLs: if the user has access to multiple datasets of the same name, use the oldest.
    * This is reasonable, because the legacy URL/NML was likely created before this ambiguity became possible */
   def getOrganizationForDataSet(dataSetName: String)(implicit ctx: DBAccessContext): Fox[ObjectId] =

diff --git a/app/models/binary/DataSetService.scala b/app/models/binary/DataSetService.scala
@@ -37,7 +37,6 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
                                teamDAO: TeamDAO,
                                workerDAO: WorkerDAO,
                                publicationDAO: PublicationDAO,
-                               publicationService: PublicationService,
                                dataStoreService: DataStoreService,
                                teamService: TeamService,
                                userService: UserService,
@@ -343,21 +342,25 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
 
   def publicWrites(dataSet: DataSet,
                    requestingUserOpt: Option[User],
-                   organization: Organization,
-                   dataStore: DataStore,
+                   organization: Option[Organization],
+                   dataStore: Option[DataStore],
                    skipResolutions: Boolean = false,
                    requestingUserTeamManagerMemberships: Option[List[TeamMembership]] = None)(
       implicit ctx: DBAccessContext): Fox[JsObject] =
     for {
+      organization <- Fox.fillOption(organization) {
+        organizationDAO.findOne(dataSet._organization) ?~> "organization.notFound"
+      }
+      dataStore <- Fox.fillOption(dataStore) {
+        dataStoreFor(dataSet)
+      }
       teams <- allowedTeamsFor(dataSet._id, requestingUserOpt) ?~> "dataset.list.fetchAllowedTeamsFailed"
       teamsJs <- Fox.serialCombined(teams)(t => teamService.publicWrites(t, Some(organization))) ?~> "dataset.list.teamWritesFailed"
       logoUrl <- logoUrlFor(dataSet, Some(organization)) ?~> "dataset.list.fetchLogoUrlFailed"
       isEditable <- isEditableBy(dataSet, requestingUserOpt, requestingUserTeamManagerMemberships) ?~> "dataset.list.isEditableCheckFailed"
       lastUsedByUser <- lastUsedTimeFor(dataSet._id, requestingUserOpt) ?~> "dataset.list.fetchLastUsedTimeFailed"
       dataStoreJs <- dataStoreService.publicWrites(dataStore) ?~> "dataset.list.dataStoreWritesFailed"
       dataSource <- dataSourceFor(dataSet, Some(organization), skipResolutions) ?~> "dataset.list.fetchDataSourceFailed"
-      publicationOpt <- Fox.runOptional(dataSet._publication)(publicationDAO.findOne(_)) ?~> "dataset.list.fetchPublicationFailed"
-      publicationJson <- Fox.runOptional(publicationOpt)(publicationService.publicWrites) ?~> "dataset.list.publicationWritesFailed"
       worker <- workerDAO.findOneByDataStore(dataStore.name).futureBox
       jobsEnabled = conf.Features.jobsEnabled && worker.nonEmpty
     } yield {
@@ -377,7 +380,6 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
         "logoUrl" -> logoUrl,
         "sortingKey" -> dataSet.sortingKey,
         "details" -> dataSet.details,
-        "publication" -> publicationJson,
         "isUnreported" -> Json.toJson(isUnreported(dataSet)),
         "isForeign" -> dataStore.isForeign,
         "jobsEnabled" -> jobsEnabled,

diff --git a/app/models/binary/Publication.scala b/app/models/binary/Publication.scala
@@ -1,8 +1,12 @@
 package models.binary
 
+import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.schema.Tables._
 import javax.inject.Inject
+import models.annotation.{AnnotationDAO, AnnotationService}
+import play.api.http.Status.NOT_FOUND
+import play.api.libs.json.Format.GenericFormat
 import play.api.libs.json.{JsObject, Json}
 import slick.jdbc.PostgresProfile.api._
 import slick.lifted.Rep
@@ -18,17 +22,32 @@ case class Publication(_id: ObjectId,
                        created: Long = System.currentTimeMillis(),
                        isDeleted: Boolean = false)
 
-class PublicationService @Inject()()(implicit ec: ExecutionContext) {
-  def publicWrites(p: Publication): Fox[JsObject] =
-    Fox.successful(
+class PublicationService @Inject()(dataSetService: DataSetService,
+                                   dataSetDAO: DataSetDAO,
+                                   annotationService: AnnotationService,
+                                   annotationDAO: AnnotationDAO)(implicit ec: ExecutionContext) {
+
+  def publicWrites(publication: Publication): Fox[JsObject] = {
+    implicit val ctx: DBAccessContext = GlobalAccessContext
+    for {
+      dataSets <- dataSetDAO.findAllByPublication(publication._id) ?~> "not found" ~> NOT_FOUND
+      annotations <- annotationDAO.findAllByPublication(publication._id) ?~> "not found" ~> NOT_FOUND
+      dataSetsJson <- Fox.serialCombined(dataSets)(d => dataSetService.publicWrites(d, None, None, None))
+      annotationsJson <- Fox.serialCombined(annotations) { annotation =>
+        annotationService.writesWithDataset(annotation)
+      }
+    } yield
       Json.obj(
-        "id" -> p._id.id,
-        "publicationDate" -> p.publicationDate,
-        "imageUrl" -> p.imageUrl,
-        "title" -> p.title,
-        "description" -> p.description,
-        "created" -> p.created
-      ))
+        "id" -> publication._id.id,
+        "publicationDate" -> publication.publicationDate,
+        "imageUrl" -> publication.imageUrl,
+        "title" -> publication.title,
+        "description" -> publication.description,
+        "created" -> publication.created,
+        "datasets" -> dataSetsJson,
+        "annotations" -> annotationsJson
+      )
+  }
 }
 
 class PublicationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext)
@@ -52,6 +71,18 @@ class PublicationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionConte
       )
     )
 
+  override def findOne(id: ObjectId)(implicit ctx: DBAccessContext): Fox[Publication] =
+    for {
+      r <- run(sql"select #$columns from #$existingCollectionName where _id = ${id.id}".as[PublicationsRow])
+      parsed <- parseFirst(r, id)
+    } yield parsed
+
+  override def findAll(implicit ctx: DBAccessContext): Fox[List[Publication]] =
+    for {
+      r <- run(sql"select #$columns from #$existingCollectionName".as[PublicationsRow])
+      parsed <- parseAll(r)
+    } yield parsed
+
   def insertOne(p: Publication): Fox[Unit] =
     for {
       _ <- run(

diff --git a/conf/evolutions/085-annotation-publication.sql b/conf/evolutions/085-annotation-publication.sql
@@ -0,0 +1,12 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+ALTER TABLE webknossos.annotations
+  ADD COLUMN _publication CHAR(24),
+  ADD CONSTRAINT publication_ref FOREIGN KEY(_publication) REFERENCES webknossos.publications(_id) DEFERRABLE;
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 85;
+
+COMMIT TRANSACTION;
diff --git a/conf/evolutions/reversions/085-annotation-publication.sql b/conf/evolutions/reversions/085-annotation-publication.sql
@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.annotations_;
+ALTER TABLE webknossos.annotations
+  DROP COLUMN _publication CHAR(24);
+
+CREATE VIEW webknossos.annotations_ AS SELECT * FROM webknossos.annotations WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 84;
+
+COMMIT TRANSACTION;
diff --git a/conf/messages b/conf/messages
@@ -363,6 +363,8 @@ job.meshFile.notAllowed.organization = Calculating mesh files is only allowed fo
 job.globalizeFloodfill.notAllowed.organization = Globalizing floodfills is only allowed for datasets of your own organization.
 job.applyMergerMode.notAllowed.organization = Applying merger mode tracings is only allowed for datasets of your own organization.
 
+publication.notFound = Publication could not be found.
+
 agglomerateSkeleton.failed=Could not generate agglomerate skeleton.
 
 unexpectedReturn=Unexpected return of parameters
diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -223,3 +223,7 @@ POST          /jobs/run/materializeVolumeAnnotation/:organizationName/:dataSetNa
 GET           /jobs/:id                                                                 controllers.JobsController.get(id: String)
 PATCH         /jobs/:id/cancel                                                          controllers.JobsController.cancel(id: String)
 POST          /jobs/:id/status                                                          controllers.WKRemoteWorkerController.updateJobStatus(key: String, id: String)
+
+# Publications
+GET           /publications                                                             controllers.PublicationController.listPublications()
+GET           /publications/:id                                                         controllers.PublicationController.read(id: String)
diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
@@ -31,6 +31,7 @@ import type {
   APIProjectProgressReport,
   APIProjectUpdater,
   APIProjectWithAssignments,
+  APIPublication,
   APIResolutionRestrictions,
   APIScript,
   APIScriptCreator,
@@ -1672,6 +1673,18 @@ export async function getMeanAndStdDevFromDataset(
   );
 }
 
+// #### Publications
+export async function getPublications(): Promise<Array<APIPublication>> {
+  const publications = await Request.receiveJSON("/api/publications");
+  assertResponseLimit(publications);
+  return publications;
+}
+
+export async function getPublication(id: string): Promise<APIPublication> {
+  const publication = await Request.receiveJSON(`/api/publications/${id}`);
+  return publication;
+}
+
 // #### Datastores
 export async function getDatastores(): Promise<Array<APIDataStore>> {
   const datastores = await Request.receiveJSON("/api/datastores");
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).

		### Postgres Evolutions:
		- [084-annotation-contributors.sql](conf/evolutions/084-annotation-contributors.sql)
		- [085-add-annotations-publicationforeignkey](conf/evolutions/085-add-annotations-publicationforeignkey.sql)