Add tags for datasets

CHANGELOG.unreleased.md

@@ -11,7 +11,14 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/21.11.0...HEAD)
 
 ### Added
--
+- Enhanced the volume fill tool to so that it operates beyond the dimensions of the current viewport. Additionally, the fill tool can also be changed to perform in 3D instead of 2D. [#5733](https://github.com/scalableminds/webknossos/pull/5733)
+- Added the possibility to load the skeletons of specific agglomerates from an agglomerate file when opening a tracing by including a mapping and agglomerate ids in the URL hash. See the [docs](https://docs.webknossos.org/webknossos/sharing.html#sharing-link-format) for further information. [#5738](https://github.com/scalableminds/webknossos/pull/5738)
+- Added a skeleton sandbox mode where a dataset can be opened and all skeleton tracing capabilities are available. However, by default changes are not saved. At any point, users can decide to copy the current state to their account. The sandbox can be accessed at `<webknossos_host>/datasets/<organization>/<dataset>/sandbox/skeleton`. In the combination with the new agglomerate skeleton loading feature this can be used to craft links that open webknossos with an activated mapping and specific agglomerates loaded on-demand. [#5738](https://github.com/scalableminds/webknossos/pull/5738)
+- The active mapping is now included in the link copied from the "Share" modal or the new "Share" button next to the dataset position. It is automatically activated for users that open the shared link. [#5738](https://github.com/scalableminds/webknossos/pull/5738)
+- A new "Segments" tab was added which replaces the old "Meshes" tab. The tab renders a list of segments within a volume annotation for the visible segmentation layer. The list "grows" while creating an annotation or browsing a dataset. For example, selecting an existing segment or drawing with a new segment id will both ensure that the segment is listed. Via right-click, meshes can be loaded for a selected segment. The mesh will be added as child to the segment. [#5696](https://github.com/scalableminds/webknossos/pull/5696)
+- For ad-hoc mesh computation and for mesh precomputation, the user can now select which quality the mesh should have (i.e., via selecting which magnification should be used). [#5696](https://github.com/scalableminds/webknossos/pull/5696)
+- The context menu in the data viewport also allows to compute an ad-hoc mesh for the selected segment. [#5696](https://github.com/scalableminds/webknossos/pull/5696)
+- Added tagging support for datasets. [#5832](https://github.com/scalableminds/webknossos/pull/5832)
 
 ### Changed
 -

MIGRATIONS.unreleased.md

@@ -9,4 +9,6 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md).
 -
 
 ### Postgres Evolutions:
+
 - [078-annotation-layers.sql](conf/evolutions/078-annotation-layers.sql)
+- [079-add-dataset-tags.sql](conf/evolutions/079-add-dataset-tags.sql)

app/controllers/DataSetController.scala

@@ -53,7 +53,8 @@ class DataSetController @Inject()(userService: UserService,
     ((__ \ 'description).readNullable[String] and
       (__ \ 'displayName).readNullable[String] and
       (__ \ 'sortingKey).readNullable[Long] and
-      (__ \ 'isPublic).read[Boolean]).tupled
+      (__ \ 'isPublic).read[Boolean] and
+      (__ \ 'tags).read[List[String]]).tupled
 
   @ApiOperation(hidden = true, value = "")
   def removeFromThumbnailCache(organizationName: String, dataSetName: String): Action[AnyContent] =
@@ -269,7 +270,7 @@ class DataSetController @Inject()(userService: UserService,
   def update(organizationName: String, dataSetName: String): Action[JsValue] = sil.SecuredAction.async(parse.json) {
     implicit request =>
       withJsonBodyUsing(dataSetPublicReads) {
-        case (description, displayName, sortingKey, isPublic) =>
+        case (description, displayName, sortingKey, isPublic, tags) =>
           for {
             dataSet <- dataSetDAO
               .findOneByNameAndOrganization(dataSetName, request.identity._organization) ?~> notFoundMessage(
@@ -281,14 +282,13 @@ class DataSetController @Inject()(userService: UserService,
                                          displayName,
                                          sortingKey.getOrElse(dataSet.created),
                                          isPublic)
+            _ <- dataSetDAO.updateTags(dataSet._id, tags)
             updated <- dataSetDAO.findOneByNameAndOrganization(dataSetName, request.identity._organization)
             _ = analyticsService.track(ChangeDatasetSettingsEvent(request.identity, updated))
             organization <- organizationDAO.findOne(updated._organization)(GlobalAccessContext)
             dataStore <- dataSetService.dataStoreFor(updated)
             js <- dataSetService.publicWrites(updated, Some(request.identity), organization, dataStore)
-          } yield {
-            Ok(Json.toJson(js))
-          }
+          } yield Ok(Json.toJson(js))
       }
   }
 

app/models/binary/DataSet.scala

@@ -46,6 +46,7 @@ case class DataSet(
     logoUrl: Option[String],
     sortingKey: Long = System.currentTimeMillis(),
     details: Option[JsObject] = None,
+    tags: Set[String] = Set.empty,
     created: Long = System.currentTimeMillis(),
     isDeleted: Boolean = false
 ) extends FoxImplicits {
@@ -104,6 +105,7 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
         r.logourl,
         r.sortingkey.getTime,
         details,
+        parseArrayTuple(r.tags).toSet,
         r.created.getTime,
         r.isdeleted
       )
@@ -230,6 +232,13 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
     } yield ()
   }
 
+  def updateTags(id: ObjectId, tags: List[String])(implicit ctx: DBAccessContext): Fox[Unit] =
+    for {
+      _ <- assertUpdateAccess(id)
+      _ <- run(
+        sqlu"update webknossos.datasets set tags = '#${writeArrayTuple(tags.map(sanitize))}' where _id = ${id.id}")
+    } yield ()
+
   def updateAdminViewConfiguration(datasetId: ObjectId, configuration: DataSetViewConfiguration)(
       implicit ctx: DBAccessContext): Fox[Unit] =
     for {
@@ -254,15 +263,16 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
     for {
       _ <- run(
         sqlu"""insert into webknossos.dataSets(_id, _dataStore, _organization, _publication, _uploader, inboxSourceHash, defaultViewConfiguration, adminViewConfiguration, description, displayName,
-                                                             isPublic, isUsable, name, scale, status, sharingToken, sortingKey, details, created, isDeleted)
+                                                             isPublic, isUsable, name, scale, status, sharingToken, sortingKey, details, tags, created, isDeleted)
                values(${d._id.id}, ${d._dataStore}, ${d._organization.id}, #${optionLiteral(d._publication.map(_.id))},
                #${optionLiteral(d._uploader.map(_.id))},
                 #${optionLiteral(d.inboxSourceHash.map(_.toString))}, #${optionLiteral(
           defaultViewConfiguration.map(sanitize))}, #${optionLiteral(adminViewConfiguration.map(sanitize))},
                 ${d.description}, ${d.displayName}, ${d.isPublic}, ${d.isUsable},
                       ${d.name}, #${optionLiteral(d.scale.map(s => writeScaleLiteral(s)))}, ${d.status
           .take(1024)}, ${d.sharingToken}, ${new java.sql.Timestamp(d.sortingKey)}, #${optionLiteral(
-          details.map(sanitize))}, ${new java.sql.Timestamp(d.created)}, ${d.isDeleted})
+          details.map(sanitize))}, '#${writeArrayTuple(d.tags.toList.map(sanitize))}', ${new java.sql.Timestamp(
+          d.created)}, ${d.isDeleted})
             """)
     } yield ()
   }

app/models/binary/DataSetService.scala

@@ -378,6 +378,7 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO,
         "isUnreported" -> Json.toJson(isUnreported(dataSet)),
         "isForeign" -> dataStore.isForeign,
         "jobsEnabled" -> jobsEnabled,
+        "tags" -> dataSet.tags
       )
     }
 }

conf/evolutions/078-add-dataset-tags.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.dataSets_;
+
+ALTER TABLE webknossos.dataSets ADD COLUMN tags VARCHAR(256)[] NOT NULL DEFAULT '{}';
+
+CREATE VIEW webknossos.dataSets_ AS SELECT * FROM webknossos.dataSets WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 78;
+
+COMMIT TRANSACTION;

conf/evolutions/079-add-dataset-tags.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.dataSets_;
+
+ALTER TABLE webknossos.dataSets ADD COLUMN tags VARCHAR(256)[] NOT NULL DEFAULT '{}';
+
+CREATE VIEW webknossos.dataSets_ AS SELECT * FROM webknossos.dataSets WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 79;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/078-add-dataset-tags.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.dataSets_;
+
+ALTER TABLE webknossos.dataSets DROP COLUMN tags;
+
+CREATE VIEW webknossos.dataSets_ AS SELECT * FROM webknossos.dataSets WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 77;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/079-add-dataset-tags.sql

@@ -0,0 +1,11 @@
+START TRANSACTION;
+
+DROP VIEW webknossos.dataSets_;
+
+ALTER TABLE webknossos.dataSets DROP COLUMN tags;
+
+CREATE VIEW webknossos.dataSets_ AS SELECT * FROM webknossos.dataSets WHERE NOT isDeleted;
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 78;
+
+COMMIT TRANSACTION;

frontend/javascripts/dashboard/advanced_dataset/dataset_action_view.js

@@ -87,7 +87,7 @@ function NewAnnotationLink({
 
 type Props = {
   dataset: APIMaybeUnimportedDataset,
-  updateDataset: APIDatasetId => Promise<void>,
+  reloadDataset: APIDatasetId => Promise<void>,
 };
 
 type State = {
@@ -131,7 +131,7 @@ class DatasetActionView extends React.PureComponent<Props, State> {
   clearCache = async (dataset: APIMaybeUnimportedDataset) => {
     this.setState({ isReloading: true });
     await clearCache(dataset);
-    await this.props.updateDataset(dataset);
+    await this.props.reloadDataset(dataset);
     Toast.success(
       messages["dataset.clear_cache_success"]({
         datasetName: dataset.name,

frontend/javascripts/dashboard/advanced_dataset/dataset_table.js

@@ -2,20 +2,29 @@
 
 import { Table, Tag } from "antd";
 import * as React from "react";
-import { CheckCircleOutlined, CloseCircleOutlined } from "@ant-design/icons";
+import { CheckCircleOutlined, CloseCircleOutlined, PlusOutlined } from "@ant-design/icons";
 import _ from "lodash";
 import { Link } from "react-router-dom";
 import dice from "dice-coefficient";
+import update from "immutability-helper";
 
-import type { APITeam, APIMaybeUnimportedDataset, APIDatasetId } from "types/api_flow_types";
+import EditableTextIcon from "oxalis/view/components/editable_text_icon";
+import type {
+  APITeam,
+  APIMaybeUnimportedDataset,
+  APIDatasetId,
+  APIDataset,
+} from "types/api_flow_types";
 import { stringToColor, formatScale } from "libs/format_utils";
 import type { DatasetFilteringMode } from "dashboard/dataset_view";
 import DatasetAccessListView from "dashboard/advanced_dataset/dataset_access_list_view";
 import DatasetActionView from "dashboard/advanced_dataset/dataset_action_view";
 import FormattedDate from "components/formatted_date";
 import { getDatasetExtentAsString } from "oxalis/model/accessors/dataset_accessor";
+import { trackAction } from "oxalis/model/helpers/analytics";
 import FixedExpandableTable from "components/fixed_expandable_table";
 import * as Utils from "libs/utils";
+import CategorizationLabel from "oxalis/view/components/categorization_label";
 
 const { Column } = Table;
 
@@ -25,11 +34,14 @@ const useLruRank = true;
 type Props = {
   datasets: Array<APIMaybeUnimportedDataset>,
   searchQuery: string,
+  searchTags: Array<string>,
   isUserAdmin: boolean,
   isUserTeamManager: boolean,
   isUserDatasetManager: boolean,
   datasetFilteringMode: DatasetFilteringMode,
-  updateDataset: (APIDatasetId, Array<APIMaybeUnimportedDataset>) => Promise<void>,
+  reloadDataset: (APIDatasetId, Array<APIMaybeUnimportedDataset>) => Promise<void>,
+  updateDataset: APIDataset => Promise<void>,
+  addTagToSearch: (tag: string) => void,
 };
 
 type State = {
@@ -68,8 +80,8 @@ class DatasetTable extends React.PureComponent<Props, State> {
     });
   };
 
-  updateSingleDataset = (datasetId: APIDatasetId): Promise<void> =>
-    this.props.updateDataset(datasetId, this.props.datasets);
+  reloadSingleDataset = (datasetId: APIDatasetId): Promise<void> =>
+    this.props.reloadDataset(datasetId, this.props.datasets);
 
   getFilteredDatasets() {
     const filterByMode = datasets => {
@@ -83,6 +95,12 @@ class DatasetTable extends React.PureComponent<Props, State> {
       }
     };
 
+    const filteredByTags = datasets =>
+      datasets.filter(dataset => {
+        const notIncludedTags = _.difference(this.props.searchTags, dataset.tags);
+        return notIncludedTags.length === 0;
+      });
+
     const filterByQuery = datasets =>
       Utils.filterWithSearchQueryAND<APIMaybeUnimportedDataset, "name" | "description">(
         datasets,
@@ -95,9 +113,33 @@ class DatasetTable extends React.PureComponent<Props, State> {
         ? datasets
         : datasets.filter(dataset => dataset.isActive && dataset.dataSource.dataLayers.length > 0);
 
-    return filterByQuery(filterByMode(filterByHasLayers(this.props.datasets)));
+    return filterByQuery(filteredByTags(filterByMode(filterByHasLayers(this.props.datasets))));
   }
 
+  editTagFromDataset = (
+    dataset: APIMaybeUnimportedDataset,
+    shouldAddTag: boolean,
+    tag: string,
+    event: SyntheticInputEvent<>,
+  ): void => {
+    event.stopPropagation(); // prevent the onClick event
+    if (!dataset.isActive) {
+      console.error(`Tags can only be added to active datasets. ${dataset.name} is not active.`);
+      return;
+    }
+    let updatedDataset = dataset;
+    if (shouldAddTag) {
+      if (!dataset.tags.includes(tag)) {
+        updatedDataset = update(dataset, { tags: { $push: [tag] } });
+      }
+    } else {
+      const newTags = _.without(dataset.tags, tag);
+      updatedDataset = update(dataset, { tags: { $set: newTags } });
+    }
+    trackAction("Edit dataset tag");
+    this.props.updateDataset(updatedDataset);
+  };
+
   renderEmptyText() {
     const maybeWarning =
       this.props.datasetFilteringMode !== "showAllDatasets" ? (
@@ -188,6 +230,37 @@ class DatasetTable extends React.PureComponent<Props, State> {
             </div>
           )}
         />
+        <Column
+          title="Tags"
+          dataIndex="tags"
+          key="tags"
+          width={280}
+          sortOrder={sortedInfo.columnKey === "name" && sortedInfo.order}
+          render={(tags: Array<string>, dataset: APIMaybeUnimportedDataset) =>
+            dataset.isActive ? (
+              <div>
+                {tags.map(tag => (
+                  <CategorizationLabel
+                    tag={tag}
+                    key={tag}
+                    kind="datasets"
+                    onClick={_.partial(this.props.addTagToSearch, tag)}
+                    onClose={_.partial(this.editTagFromDataset, dataset, false, tag)}
+                    closable
+                  />
+                ))}
+                <EditableTextIcon
+                  icon={<PlusOutlined />}
+                  onChange={_.partial(this.editTagFromDataset, dataset, true)}
+                />
+              </div>
+            ) : (
+              <div style={{ color: "@disabled-color" }}>
+                Not tags available for inactive datasets.
+              </div>
+            )
+          }
+        />
         <Column
           title="Voxel Size & Extent"
           dataIndex="scale"
@@ -276,7 +349,7 @@ class DatasetTable extends React.PureComponent<Props, State> {
           key="actions"
           fixed="right"
           render={(__, dataset: APIMaybeUnimportedDataset) => (
-            <DatasetActionView dataset={dataset} updateDataset={this.updateSingleDataset} />
+            <DatasetActionView dataset={dataset} reloadDataset={this.reloadSingleDataset} />
           )}
         />
       </FixedExpandableTable>