diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index d29d399b28f..fb2aa1a0c4d 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -12,6 +12,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Added - The target folder of a dataset can now be specified during upload. Also, clicking "Add Dataset" from an active folder will upload the dataset to that folder by default. [#6704](https://github.com/scalableminds/webknossos/pull/6704) +- The storage used by an organization’s datasets can now be measured. [#6685](https://github.com/scalableminds/webknossos/pull/6685) ### Changed - Improved performance of opening a dataset or annotation. [#6711](https://github.com/scalableminds/webknossos/pull/6711) diff --git a/MIGRATIONS.unreleased.md b/MIGRATIONS.unreleased.md index ccd8c2fd4a4..350a77b77d7 100644 --- a/MIGRATIONS.unreleased.md +++ b/MIGRATIONS.unreleased.md @@ -12,3 +12,4 @@ User-facing changes are documented in the [changelog](CHANGELOG.released.md). - [094-pricing-plans.sql](conf/evolutions/reversions/094-pricing-plans.sql) - [095-constraint-naming.sql](conf/evolutions/reversions/095-constraint-naming.sql) +- [096-storage.sql](conf/evolutions/096-storage.sql) diff --git a/app/WebKnossosModule.scala b/app/WebKnossosModule.scala index 084e0fc0b4c..024f750f26f 100644 --- a/app/WebKnossosModule.scala +++ b/app/WebKnossosModule.scala @@ -4,6 +4,7 @@ import models.analytics.AnalyticsSessionService import models.annotation.AnnotationStore import models.binary.DataSetService import models.job.{JobService, WorkerLivenessService} +import models.storage.UsedStorageService import models.task.TaskService import models.user.time.TimeSpanService import models.user._ @@ -34,5 +35,6 @@ class WebKnossosModule extends AbstractModule { bind(classOf[AnalyticsSessionService]).asEagerSingleton() bind(classOf[WorkerLivenessService]).asEagerSingleton() bind(classOf[ElasticsearchClient]).asEagerSingleton() + bind(classOf[UsedStorageService]).asEagerSingleton() } } diff --git a/app/controllers/DataSetController.scala b/app/controllers/DataSetController.scala index d5828e8e96d..7ab2f731741 100755 --- a/app/controllers/DataSetController.scala +++ b/app/controllers/DataSetController.scala @@ -99,6 +99,7 @@ class DataSetController @Inject()(userService: UserService, .clientFor(dataSet)(GlobalAccessContext) .flatMap( _.requestDataLayerThumbnail(organizationName, + dataSet, dataLayerName, width, height, @@ -320,7 +321,7 @@ class DataSetController @Inject()(userService: UserService, datalayer <- usableDataSource.dataLayers.headOption.toFox ?~> "dataSet.noLayers" _ <- dataSetService .clientFor(dataSet)(GlobalAccessContext) - .flatMap(_.findPositionWithData(organizationName, datalayer.name).flatMap(posWithData => + .flatMap(_.findPositionWithData(organizationName, dataSet, datalayer.name).flatMap(posWithData => bool2Fox(posWithData.value("position") != JsNull))) ?~> "dataSet.loadingDataFailed" } yield { Ok("Ok") diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala index d95faedcbfe..871e10c2be8 100644 --- a/app/controllers/WKRemoteDataStoreController.scala +++ b/app/controllers/WKRemoteDataStoreController.scala @@ -16,6 +16,7 @@ import models.binary._ import models.folder.FolderDAO import models.job.JobDAO import models.organization.OrganizationDAO +import models.storage.UsedStorageService import models.user.{User, UserDAO, UserService} import net.liftweb.common.Full import oxalis.mail.{MailchimpClient, MailchimpTag} @@ -35,6 +36,7 @@ class WKRemoteDataStoreController @Inject()( analyticsService: AnalyticsService, userService: UserService, organizationDAO: OrganizationDAO, + usedStorageService: UsedStorageService, dataSetDAO: DataSetDAO, userDAO: UserDAO, folderDAO: FolderDAO, @@ -95,6 +97,7 @@ class WKRemoteDataStoreController @Inject()( dataSet <- dataSetDAO.findOneByNameAndOrganization(dataSetName, user._organization)(GlobalAccessContext) ?~> Messages( "dataSet.notFound", dataSetName) ~> NOT_FOUND + _ <- usedStorageService.refreshStorageReportForDataset(dataSet) _ = analyticsService.track(UploadDatasetEvent(user, dataSet, dataStore, dataSetSizeBytes)) _ = mailchimpClient.tagUser(user, MailchimpTag.HasUploadedOwnDataset) } yield Ok @@ -106,7 +109,11 @@ class WKRemoteDataStoreController @Inject()( request.body.validate[DataStoreStatus] match { case JsSuccess(status, _) => logger.debug(s"Status update from data store '$name'. Status: " + status.ok) - dataStoreDAO.updateUrlByName(name, status.url).map(_ => Ok) + for { + _ <- dataStoreDAO.updateUrlByName(name, status.url) + _ <- dataStoreDAO.updateReportUsedStorageEnabledByName(name, + status.reportUsedStorageEnabled.getOrElse(false)) + } yield Ok case e: JsError => logger.error("Data store '$name' sent invalid update. Error: " + e) Future.successful(JsonBadRequest(JsError.toJson(e))) @@ -159,8 +166,12 @@ class WKRemoteDataStoreController @Inject()( .findOneByNameAndOrganizationName(datasourceId.name, datasourceId.team)(GlobalAccessContext) .futureBox _ <- existingDataset.flatMap { - case Full(dataset) => dataSetDAO.deleteDataset(dataset._id) - case _ => Fox.successful(()) + case Full(dataset) => { + dataSetDAO + .deleteDataset(dataset._id) + .flatMap(_ => usedStorageService.refreshStorageReportForDataset(dataset)) + } + case _ => Fox.successful(()) } } yield Ok } diff --git a/app/models/binary/DataSetService.scala b/app/models/binary/DataSetService.scala index a7eec7ed402..e82cd5566d1 100644 --- a/app/models/binary/DataSetService.scala +++ b/app/models/binary/DataSetService.scala @@ -256,7 +256,7 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO, def clientFor(dataSet: DataSet)(implicit ctx: DBAccessContext): Fox[WKRemoteDataStoreClient] = for { dataStore <- dataStoreFor(dataSet) - } yield new WKRemoteDataStoreClient(dataStore, dataSet, rpc) + } yield new WKRemoteDataStoreClient(dataStore, rpc) def lastUsedTimeFor(_dataSet: ObjectId, userOpt: Option[User]): Fox[Instant] = userOpt match { diff --git a/app/models/binary/DataStore.scala b/app/models/binary/DataStore.scala index fdf10e6a1b6..4d73ef14ff0 100644 --- a/app/models/binary/DataStore.scala +++ b/app/models/binary/DataStore.scala @@ -23,6 +23,7 @@ case class DataStore( isDeleted: Boolean = false, isConnector: Boolean = false, allowsUpload: Boolean = true, + reportUsedStorageEnabled: Boolean = false, onlyAllowedOrganization: Option[ObjectId] = None ) @@ -45,6 +46,7 @@ object DataStore { isDeleted = false, isConnector.getOrElse(false), allowsUpload.getOrElse(true), + reportUsedStorageEnabled = false, None ) @@ -102,6 +104,7 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext r.isdeleted, r.isconnector, r.allowsupload, + r.reportusedstorageenabled, r.onlyallowedorganization.map(ObjectId(_)) )) @@ -126,16 +129,28 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext parsed <- parseAll(r) } yield parsed + def findAllWithStorageReporting: Fox[List[DataStore]] = + for { + r <- run(sql"select #$columns from webknossos.datastores_ where reportUsedStorageEnabled".as[DatastoresRow]) + parsed <- parseAll(r) + } yield parsed + def updateUrlByName(name: String, url: String): Fox[Unit] = { val q = for { row <- Datastores if notdel(row) && row.name === name } yield row.url for { _ <- run(q.update(url)) } yield () } + def updateReportUsedStorageEnabledByName(name: String, reportUsedStorageEnabled: Boolean): Fox[Unit] = + for { + _ <- run( + sqlu"UPDATE webknossos.dataStores SET reportUsedStorageEnabled = $reportUsedStorageEnabled WHERE name = $name") + } yield () + def insertOne(d: DataStore): Fox[Unit] = for { _ <- run( - sqlu"""insert into webknossos.dataStores(name, url, publicUrl, key, isScratch, isDeleted, isConnector, allowsUpload) - values(${d.name}, ${d.url}, ${d.publicUrl}, ${d.key}, ${d.isScratch}, ${d.isDeleted}, ${d.isConnector}, ${d.allowsUpload})""") + sqlu"""insert into webknossos.dataStores(name, url, publicUrl, key, isScratch, isDeleted, isConnector, allowsUpload, reportUsedStorageEnabled) + values(${d.name}, ${d.url}, ${d.publicUrl}, ${d.key}, ${d.isScratch}, ${d.isDeleted}, ${d.isConnector}, ${d.allowsUpload}, ${d.reportUsedStorageEnabled})""") } yield () def deleteOneByName(name: String): Fox[Unit] = diff --git a/app/models/binary/WKRemoteDataStoreClient.scala b/app/models/binary/WKRemoteDataStoreClient.scala index f84ae81668a..da00dcabaf4 100644 --- a/app/models/binary/WKRemoteDataStoreClient.scala +++ b/app/models/binary/WKRemoteDataStoreClient.scala @@ -3,16 +3,16 @@ package models.binary import com.scalableminds.util.geometry.Vec3Int import com.scalableminds.util.tools.Fox import com.scalableminds.webknossos.datastore.rpc.RPC +import com.scalableminds.webknossos.datastore.services.DirectoryStorageReport import com.typesafe.scalalogging.LazyLogging import controllers.RpcTokenHolder import play.api.libs.json.JsObject import play.utils.UriEncoding -class WKRemoteDataStoreClient(dataStore: DataStore, dataSet: DataSet, rpc: RPC) extends LazyLogging { - - def baseInfo = s"Dataset: ${dataSet.name} Datastore: ${dataStore.url}" +class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLogging { def requestDataLayerThumbnail(organizationName: String, + dataSet: DataSet, dataLayerName: String, width: Int, height: Int, @@ -29,7 +29,7 @@ class WKRemoteDataStoreClient(dataStore: DataStore, dataSet: DataSet, rpc: RPC) .getWithBytesResponse } - def findPositionWithData(organizationName: String, dataLayerName: String): Fox[JsObject] = + def findPositionWithData(organizationName: String, dataSet: DataSet, dataLayerName: String): Fox[JsObject] = rpc( s"${dataStore.url}/data/datasets/${urlEncode(organizationName)}/${dataSet.urlEncodedName}/layers/$dataLayerName/findData") .addQueryString("token" -> RpcTokenHolder.webKnossosToken) @@ -37,4 +37,11 @@ class WKRemoteDataStoreClient(dataStore: DataStore, dataSet: DataSet, rpc: RPC) private def urlEncode(text: String) = UriEncoding.encodePathSegment(text, "UTF-8") + def fetchStorageReport(organizationName: String, datasetName: Option[String]): Fox[List[DirectoryStorageReport]] = + rpc(s"${dataStore.url}/data/datasets/measureUsedStorage/${urlEncode(organizationName)}") + .addQueryString("token" -> RpcTokenHolder.webKnossosToken) + .addQueryStringOptional("dataSetName", datasetName) + .silent + .getWithJsonResponse[List[DirectoryStorageReport]] + } diff --git a/app/models/organization/Organization.scala b/app/models/organization/Organization.scala index 600b66fe925..f00e2a98f25 100755 --- a/app/models/organization/Organization.scala +++ b/app/models/organization/Organization.scala @@ -3,9 +3,8 @@ package models.organization import com.scalableminds.util.accesscontext.DBAccessContext import com.scalableminds.util.time.Instant import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.services.DirectoryStorageReport import com.scalableminds.webknossos.schema.Tables._ - -import javax.inject.Inject import models.team.PricingPlan import models.team.PricingPlan.PricingPlan import slick.jdbc.PostgresProfile.api._ @@ -13,7 +12,9 @@ import slick.lifted.Rep import utils.sql.{SQLClient, SQLDAO} import utils.ObjectId +import javax.inject.Inject import scala.concurrent.ExecutionContext +import scala.concurrent.duration.FiniteDuration case class Organization( _id: ObjectId, @@ -24,7 +25,7 @@ case class Organization( pricingPlan: PricingPlan, paidUntil: Option[Instant], includedUsers: Option[Int], // None means unlimited - includedStorage: Option[Long], // None means unlimited + includedStorageBytes: Option[Long], // None means unlimited _rootFolder: ObjectId, newUserMailingList: String = "", overTimeMailingList: String = "", @@ -101,7 +102,7 @@ class OrganizationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionCont VALUES (${o._id.id}, ${o.name}, ${o.additionalInformation}, ${o.logoUrl}, ${o.displayName}, ${o._rootFolder}, ${o.newUserMailingList}, ${o.overTimeMailingList}, ${o.enableAutoVerify}, - '#${o.pricingPlan}', ${o.paidUntil}, ${o.includedUsers}, ${o.includedStorage}, ${o.lastTermsOfServiceAcceptanceTime}, + '#${o.pricingPlan}', ${o.paidUntil}, ${o.includedUsers}, ${o.includedStorageBytes}, ${o.lastTermsOfServiceAcceptanceTime}, ${o.lastTermsOfServiceAcceptanceVersion}, ${o.created}, ${o.isDeleted}) """) } yield () @@ -132,6 +133,68 @@ class OrganizationDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionCont where _id = $organizationId""") } yield () + def deleteUsedStorage(organizationId: ObjectId): Fox[Unit] = + for { + _ <- run(sqlu"DELETE FROM webknossos.organization_usedStorage WHERE _organization = $organizationId") + } yield () + + def deleteUsedStorageForDataset(datasetId: ObjectId): Fox[Unit] = + for { + _ <- run(sqlu"DELETE FROM webknossos.organization_usedStorage WHERE _dataSet = $datasetId") + } yield () + + def updateLastStorageScanTime(organizationId: ObjectId, time: Instant): Fox[Unit] = + for { + _ <- run(sqlu"UPDATE webknossos.organizations SET lastStorageScanTime = $time WHERE _id = $organizationId") + } yield () + + def upsertUsedStorage(organizationId: ObjectId, + dataStoreName: String, + usedStorageEntries: List[DirectoryStorageReport]): Fox[Unit] = { + val queries = usedStorageEntries.map(entry => sqlu""" + WITH ds AS ( + SELECT _id + FROM webknossos.datasets_ + WHERE _organization = $organizationId + AND name = ${entry.dataSetName} + LIMIT 1 + ) + INSERT INTO webknossos.organization_usedStorage( + _organization, _dataStore, _dataSet, layerName, + magOrDirectoryName, usedStorageBytes, lastUpdated) + SELECT + $organizationId, $dataStoreName, ds._id, ${entry.layerName}, + ${entry.magOrDirectoryName}, ${entry.usedStorageBytes}, NOW() + FROM ds + ON CONFLICT (_organization, _dataStore, _dataSet, layerName, magOrDirectoryName) + DO UPDATE + SET usedStorageBytes = ${entry.usedStorageBytes}, lastUpdated = NOW() + """) + for { + _ <- Fox.serialCombined(queries)(q => run(q)) + } yield () + } + + def getUsedStorage(organizationId: ObjectId): Fox[Long] = + for { + rows <- run( + sql"SELECT SUM(usedStorageBytes) FROM webknossos.organization_usedStorage WHERE _organization = $organizationId" + .as[Long]) + firstRow <- rows.headOption + } yield firstRow + + def findNotRecentlyScanned(scanInterval: FiniteDuration, limit: Int): Fox[List[Organization]] = + for { + rows <- run(sql""" + SELECT #$columns + FROM #$existingCollectionName + WHERE lastStorageScanTime < ${Instant.now - scanInterval} + ORDER BY lastStorageScanTime + LIMIT $limit + """.as[OrganizationsRow]) + parsed <- parseAll(rows) + } yield parsed + def acceptTermsOfService(organizationId: ObjectId, version: Int, timestamp: Long)( implicit ctx: DBAccessContext): Fox[Unit] = for { diff --git a/app/models/organization/OrganizationService.scala b/app/models/organization/OrganizationService.scala index c3daf0399d6..867a0b829d7 100644 --- a/app/models/organization/OrganizationService.scala +++ b/app/models/organization/OrganizationService.scala @@ -36,7 +36,9 @@ class OrganizationService @Inject()(organizationDAO: OrganizationDAO, "lastTermsOfServiceAcceptanceVersion" -> organization.lastTermsOfServiceAcceptanceVersion ) } else Json.obj() - Fox.successful( + for { + usedStorageBytes <- organizationDAO.getUsedStorage(organization._id) + } yield Json.obj( "id" -> organization._id.toString, "name" -> organization.name, @@ -46,9 +48,9 @@ class OrganizationService @Inject()(organizationDAO: OrganizationDAO, "pricingPlan" -> organization.pricingPlan, "paidUntil" -> organization.paidUntil, "includedUsers" -> organization.includedUsers, - "includedStorage" -> organization.includedStorage.map(bytes => bytes / 1000000) + "includedStorageBytes" -> organization.includedStorageBytes, + "usedStorageBytes" -> usedStorageBytes ) ++ adminOnlyInfo - ) } def findOneByInviteByNameOrDefault(inviteOpt: Option[Invite], organizationNameOpt: Option[String])( diff --git a/app/models/storage/UsedStorageService.scala b/app/models/storage/UsedStorageService.scala new file mode 100644 index 00000000000..88175b22578 --- /dev/null +++ b/app/models/storage/UsedStorageService.scala @@ -0,0 +1,94 @@ +package models.storage + +import akka.actor.ActorSystem +import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext} +import com.scalableminds.util.time.Instant +import com.scalableminds.util.tools.Fox +import com.scalableminds.webknossos.datastore.helpers.IntervalScheduler +import com.scalableminds.webknossos.datastore.rpc.RPC +import com.scalableminds.webknossos.datastore.services.DirectoryStorageReport +import com.typesafe.scalalogging.LazyLogging +import models.binary.{DataSet, DataSetService, DataStore, DataStoreDAO, WKRemoteDataStoreClient} +import models.organization.{Organization, OrganizationDAO} +import play.api.inject.ApplicationLifecycle +import utils.WkConf + +import javax.inject.Inject +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +class UsedStorageService @Inject()(val system: ActorSystem, + val lifecycle: ApplicationLifecycle, + organizationDAO: OrganizationDAO, + dataSetService: DataSetService, + dataStoreDAO: DataStoreDAO, + rpc: RPC, + config: WkConf)(implicit ec: ExecutionContext) + extends LazyLogging + with IntervalScheduler { + + /* Note that not every tick here will scan something, there is additional logic below: + Every tick, et most 10 organizations are scanned. But only if their last full scan is was sufficiently long ago + The 10 organizations with the most outdated scan are selected each tick. This is to distribute the load. + */ + override protected def tickerInterval: FiniteDuration = 10 minutes + override protected def tickerInitialDelay: FiniteDuration = 1 minute + + private val isRunning = new java.util.concurrent.atomic.AtomicBoolean(false) + + private val pauseAfterEachOrganization = 5 seconds + private val organizationCountToScanPerTick = 10 + + implicit private val ctx: DBAccessContext = GlobalAccessContext + + override protected def tick(): Unit = + if (isRunning.compareAndSet(false, true)) { + tickAsync().futureBox.onComplete { _ => + isRunning.set(false) + } + } + + private def tickAsync(): Fox[Unit] = + for { + organizations <- organizationDAO.findNotRecentlyScanned(config.WebKnossos.FetchUsedStorage.interval, + organizationCountToScanPerTick) + dataStores <- dataStoreDAO.findAllWithStorageReporting + _ = logger.info(s"Scanning ${organizations.length} organizations in ${dataStores.length} datastores...") + _ <- Fox.serialCombined(organizations)(organization => refreshStorageReports(organization, dataStores)) + } yield () + + private def refreshStorageReports(organization: Organization, dataStores: List[DataStore]): Fox[Unit] = + for { + storageReportsByDataStore <- Fox.serialCombined(dataStores)(dataStore => + refreshStorageReports(dataStore, organization)) + _ <- organizationDAO.deleteUsedStorage(organization._id) + _ <- Fox.serialCombined(storageReportsByDataStore.zip(dataStores))(storageForDatastore => + upsertUsedStorage(organization, storageForDatastore)) + _ <- organizationDAO.updateLastStorageScanTime(organization._id, Instant.now) + _ = Thread.sleep(pauseAfterEachOrganization.toMillis) + } yield () + + private def refreshStorageReports(dataStore: DataStore, + organization: Organization): Fox[List[DirectoryStorageReport]] = { + val dataStoreClient = new WKRemoteDataStoreClient(dataStore, rpc) + dataStoreClient.fetchStorageReport(organization.name, datasetName = None) + } + + private def upsertUsedStorage(organization: Organization, + storageReportsForDatastore: (List[DirectoryStorageReport], DataStore)): Fox[Unit] = { + val dataStore = storageReportsForDatastore._2 + val storageReports = storageReportsForDatastore._1 + organizationDAO.upsertUsedStorage(organization._id, dataStore.name, storageReports) + } + + def refreshStorageReportForDataset(dataset: DataSet): Fox[Unit] = + for { + dataStore <- dataSetService.dataStoreFor(dataset) + dataStoreClient = new WKRemoteDataStoreClient(dataStore, rpc) + organization <- organizationDAO.findOne(dataset._organization) + report <- dataStoreClient.fetchStorageReport(organization.name, Some(dataset.name)) + _ <- organizationDAO.deleteUsedStorageForDataset(dataset._id) + _ <- organizationDAO.upsertUsedStorage(organization._id, dataStore.name, report) + } yield () + +} diff --git a/app/utils/WkConf.scala b/app/utils/WkConf.scala index 0cd40e467e7..e665421789b 100644 --- a/app/utils/WkConf.scala +++ b/app/utils/WkConf.scala @@ -62,6 +62,10 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L val children = List(User) } + object FetchUsedStorage { + val interval: FiniteDuration = get[FiniteDuration]("webKnossos.fetchUsedStorage.interval") + } + object TermsOfService { val enabled: Boolean = get[Boolean]("webKnossos.termsOfService.enabled") val url: String = get[String]("webKnossos.termsOfService.url") @@ -70,7 +74,7 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L } val operatorData: String = get[String]("webKnossos.operatorData") - val children = List(User, Tasks, Cache, SampleOrganization) + val children = List(User, Tasks, Cache, SampleOrganization, FetchUsedStorage, TermsOfService) } object SingleSignOn { diff --git a/app/utils/sql/SQLHelpers.scala b/app/utils/sql/SQLHelpers.scala index 075c454b9a1..6ab7b3079c1 100644 --- a/app/utils/sql/SQLHelpers.scala +++ b/app/utils/sql/SQLHelpers.scala @@ -13,9 +13,12 @@ import slick.dbio.DBIOAction import slick.jdbc.PostgresProfile.api._ import slick.jdbc._ import slick.lifted.{AbstractTable, Rep, TableQuery} +import slick.util.{Dumpable, TreePrinter} import utils.ObjectId import utils.sql.SqlInterpolation.sqlInterpolation +import java.io.{ByteArrayOutputStream, PrintWriter} +import java.nio.charset.StandardCharsets import javax.inject.Inject import scala.annotation.nowarn import scala.concurrent.ExecutionContext @@ -163,16 +166,23 @@ class SimpleSQLDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext foxFuture.toFox.flatten } + private def querySummary(query: Dumpable): String = { + val treePrinter = new TreePrinter() + val os = new ByteArrayOutputStream() + treePrinter.print(query, new PrintWriter(os)) + new String(os.toByteArray, StandardCharsets.UTF_8) + } + private def logError[R](ex: Throwable, query: DBIOAction[R, NoStream, Nothing]): Unit = { logger.error("SQL Error: " + ex) - logger.debug("Caused by query:\n" + query.getDumpInfo.mainInfo) + logger.debug("Caused by query:\n" + querySummary(query).take(4000)) } private def reportErrorToSlack[R](ex: Throwable, query: DBIOAction[R, NoStream, Nothing]): Unit = sqlClient.getSlackNotificationService.warnWithException( "SQL Error", ex, - s"Causing query: ${query.getDumpInfo.mainInfo}" + s"Causing query: ${querySummary(query).take(4000)}" ) } diff --git a/conf/application.conf b/conf/application.conf index b6950491e75..be9158557d0 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -59,6 +59,9 @@ webKnossos { cache { user.timeout = 3 minutes } + fetchUsedStorage { + interval = 24 hours # note that this interval works across wk restarts + } sampleOrganization { enabled = true user { @@ -149,6 +152,7 @@ datastore { enabled = true interval = 1 minute } + reportUsedStorage.enabled = false cache { dataCube.maxEntries = 40 mapping.maxEntries = 5 diff --git a/conf/evolutions/096-storage.sql b/conf/evolutions/096-storage.sql new file mode 100644 index 00000000000..cc374364556 --- /dev/null +++ b/conf/evolutions/096-storage.sql @@ -0,0 +1,37 @@ +BEGIN transaction; + +DROP VIEW webknossos.userInfos; +DROP VIEW webknossos.dataStores_; +DROP VIEW webknossos.organizations_; + +ALTER TABLE webknossos.dataStores ADD COLUMN reportUsedStorageEnabled BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE webknossos.organizations ADD COLUMN lastStorageScanTime TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01T00:00:00.000Z'; + +CREATE TABLE webknossos.organization_usedStorage( + _organization CHAR(24) NOT NULL, + _dataStore VARCHAR(256) NOT NULL, + _dataSet CHAR(24) NOT NULL, + layerName VARCHAR(256) NOT NULL, + magOrDirectoryName VARCHAR(256) NOT NULL, + usedStorageBytes BIGINT NOT NULL, + lastUpdated TIMESTAMPTZ, + PRIMARY KEY(_organization, _dataStore, _dataSet, layerName, magOrDirectoryName) +); + +-- recreate dropped views +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; +CREATE VIEW webknossos.organizations_ AS SELECT * FROM webknossos.organizations WHERE NOT isDeleted; + +CREATE VIEW webknossos.userInfos AS +SELECT +u._id AS _user, m.email, u.firstName, u.lastname, o.displayName AS organization_displayName, +u.isDeactivated, u.isDatasetManager, u.isAdmin, m.isSuperUser, +u._organization, o.name AS organization_name, u.created AS user_created, +m.created AS multiuser_created, u._multiUser, m._lastLoggedInIdentity, u.lastActivity +FROM webknossos.users_ u +JOIN webknossos.organizations_ o ON u._organization = o._id +JOIN webknossos.multiUsers_ m on u._multiUser = m._id; + +UPDATE webknossos.releaseInformation SET schemaVersion = 96; + +COMMIT; diff --git a/conf/evolutions/reversions/096-storage.sql b/conf/evolutions/reversions/096-storage.sql new file mode 100644 index 00000000000..6fe6d7d3d86 --- /dev/null +++ b/conf/evolutions/reversions/096-storage.sql @@ -0,0 +1,28 @@ +BEGIN transaction; + +DROP VIEW webknossos.userInfos; +DROP VIEW webknossos.dataStores_; +DROP VIEW webknossos.organizations_; + +ALTER TABLE webknossos.dataStores DROP COLUMN reportUsedStorageEnabled; +ALTER TABLE webknossos.organizations DROP COLUMN lastStorageScanTime; + +DROP TABLE webknossos.organization_usedStorage; + +-- recreate dropped views +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; +CREATE VIEW webknossos.organizations_ AS SELECT * FROM webknossos.organizations WHERE NOT isDeleted; + +CREATE VIEW webknossos.userInfos AS +SELECT +u._id AS _user, m.email, u.firstName, u.lastname, o.displayName AS organization_displayName, +u.isDeactivated, u.isDatasetManager, u.isAdmin, m.isSuperUser, +u._organization, o.name AS organization_name, u.created AS user_created, +m.created AS multiuser_created, u._multiUser, m._lastLoggedInIdentity, u.lastActivity +FROM webknossos.users_ u +JOIN webknossos.organizations_ o ON u._organization = o._id +JOIN webknossos.multiUsers_ m on u._multiUser = m._id; + +UPDATE webknossos.releaseInformation SET schemaVersion = 95; + +COMMIT; diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 55e846a322e..0181fae9849 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -63,7 +63,6 @@ import type { VoxelyticsWorkflowReport, VoxelyticsChunkStatistics, ShortLink, - APIOrganizationStorageInfo, APIPricingPlanStatus, } from "types/api_flow_types"; import { APIAnnotationTypeEnum } from "types/api_flow_types"; @@ -1938,7 +1937,7 @@ export async function getOrganization(organizationName: string): Promise { - // TODO switch to a real API. See PR #6614 - const usedStorageMB = 0; - return Promise.resolve({ usedStorageSpace: usedStorageMB }); -} - export async function sendUpgradePricingPlanEmail(requestedPlan: string): Promise { return Request.receiveJSON(`/api/pricing/requestUpgrade?requestedPlan=${requestedPlan}`, { method: "POST", diff --git a/frontend/javascripts/admin/organization/organization_cards.tsx b/frontend/javascripts/admin/organization/organization_cards.tsx index 26ff17936bc..de9b40012f0 100644 --- a/frontend/javascripts/admin/organization/organization_cards.tsx +++ b/frontend/javascripts/admin/organization/organization_cards.tsx @@ -173,32 +173,31 @@ export function PlanExpirationCard({ organization }: { organization: APIOrganiza export function PlanDashboardCard({ organization, activeUsersCount, - usedStorageSpace, }: { organization: APIOrganization; activeUsersCount: number; - usedStorageSpace: number; }) { const usedUsersPercentage = (activeUsersCount / organization.includedUsers) * 100; - const usedStoragePercentage = (usedStorageSpace / organization.includedStorage) * 100; + const usedStoragePercentage = + (organization.usedStorageBytes / organization.includedStorageBytes) * 100; const hasExceededUserLimit = hasPricingPlanExceededUsers(organization, activeUsersCount); - const hasExceededStorageLimit = hasPricingPlanExceededStorage(organization, usedStorageSpace); + const hasExceededStorageLimit = hasPricingPlanExceededStorage(organization); const maxUsersCountLabel = organization.includedUsers === Number.POSITIVE_INFINITY ? "∞" : organization.includedUsers; let includedStorageLabel = organization.pricingPlan === PricingPlanEnum.Basic - ? `${(organization.includedStorage / 1000).toFixed(0)}GB` - : `${(organization.includedStorage / 1000 ** 2).toFixed(0)}TB`; + ? `${(organization.includedStorageBytes / 10 ** 9).toFixed(0)}GB` + : `${(organization.includedStorageBytes / 10 ** 12).toFixed(0)}TB`; includedStorageLabel = - organization.includedStorage === Number.POSITIVE_INFINITY ? "∞" : includedStorageLabel; + organization.includedStorageBytes === Number.POSITIVE_INFINITY ? "∞" : includedStorageLabel; const usedStorageLabel = organization.pricingPlan === PricingPlanEnum.Basic - ? `${(usedStorageSpace / 1000).toFixed(1)}` - : `${(usedStorageSpace / 1000 ** 2).toFixed(1)}`; + ? `${(organization.usedStorageBytes / 10 ** 9).toFixed(1)}` + : `${(organization.usedStorageBytes / 10 ** 12).toFixed(1)}`; const storageLabel = `${usedStorageLabel}/${includedStorageLabel}`; diff --git a/frontend/javascripts/admin/organization/organization_edit_view.tsx b/frontend/javascripts/admin/organization/organization_edit_view.tsx index 5718c3fd254..931098ba943 100644 --- a/frontend/javascripts/admin/organization/organization_edit_view.tsx +++ b/frontend/javascripts/admin/organization/organization_edit_view.tsx @@ -15,7 +15,6 @@ import { updateOrganization, getUsers, getPricingPlanStatus, - getOrganizationStorageSpace, } from "admin/admin_rest_api"; import Toast from "libs/toast"; import { coalesce } from "libs/utils"; @@ -50,7 +49,6 @@ type State = { organization: APIOrganization | null; activeUsersCount: number; pricingPlanStatus: APIPricingPlanStatus | null; - usedStorageSpace: number | null; }; class OrganizationEditView extends React.PureComponent { @@ -63,7 +61,6 @@ class OrganizationEditView extends React.PureComponent { organization: null, activeUsersCount: 1, pricingPlanStatus: null, - usedStorageSpace: null, }; formRef = React.createRef(); @@ -101,11 +98,10 @@ class OrganizationEditView extends React.PureComponent { this.setState({ isFetchingData: true, }); - const [organization, users, pricingPlanStatus, usedStorageSpace] = await Promise.all([ + const [organization, users, pricingPlanStatus] = await Promise.all([ getOrganization(this.props.organizationName), getUsers(), getPricingPlanStatus(), - getOrganizationStorageSpace(this.props.organizationName), ]); const { displayName, newUserMailingList, pricingPlan } = organization; @@ -117,7 +113,6 @@ class OrganizationEditView extends React.PureComponent { organization, pricingPlanStatus, activeUsersCount: getActiveUserCount(users), - usedStorageSpace: usedStorageSpace.usedStorageSpace, }); } @@ -165,8 +160,7 @@ class OrganizationEditView extends React.PureComponent { this.state.isFetchingData || !this.state.organization || !this.state.pricingPlan || - !this.state.pricingPlanStatus || - this.state.usedStorageSpace === null + !this.state.pricingPlanStatus ) return (
{ diff --git a/frontend/javascripts/admin/organization/pricing_plan_utils.ts b/frontend/javascripts/admin/organization/pricing_plan_utils.ts index defa27dc889..635ae1b437d 100644 --- a/frontend/javascripts/admin/organization/pricing_plan_utils.ts +++ b/frontend/javascripts/admin/organization/pricing_plan_utils.ts @@ -1,4 +1,4 @@ -import { APIOrganization, APIOrganizationStorageInfo, APIUser } from "types/api_flow_types"; +import { APIOrganization, APIUser } from "types/api_flow_types"; import { PricingPlanEnum } from "./organization_edit_view"; export const teamPlanFeatures = [ @@ -34,11 +34,8 @@ export function hasPricingPlanExceededUsers( return activeUserCount > organization.includedUsers; } -export function hasPricingPlanExceededStorage( - organization: APIOrganization, - usedStorageSpaceMB: number, -): boolean { - return usedStorageSpaceMB > organization.includedStorage; +export function hasPricingPlanExceededStorage(organization: APIOrganization): boolean { + return organization.usedStorageBytes > organization.includedStorageBytes; } export function isUserAllowedToRequestUpgrades(user: APIUser): boolean { diff --git a/frontend/javascripts/dashboard/dashboard_view.tsx b/frontend/javascripts/dashboard/dashboard_view.tsx index 63fbc1998ac..efe38ae16cc 100644 --- a/frontend/javascripts/dashboard/dashboard_view.tsx +++ b/frontend/javascripts/dashboard/dashboard_view.tsx @@ -7,17 +7,11 @@ import React, { PureComponent, useContext } from "react"; import _ from "lodash"; import { setActiveUserAction } from "oxalis/model/actions/user_actions"; import { WhatsNextHeader } from "admin/welcome_ui"; -import type { - APIOrganization, - APIOrganizationStorageInfo, - APIPricingPlanStatus, - APIUser, -} from "types/api_flow_types"; +import type { APIOrganization, APIPricingPlanStatus, APIUser } from "types/api_flow_types"; import type { OxalisState } from "oxalis/store"; import { enforceActiveUser } from "oxalis/model/accessors/user_accessor"; import { getOrganization, - getOrganizationStorageSpace, getPricingPlanStatus, getUser, updateNovelUserExperienceInfos, diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index 6e83d9bc842..5541ba819e4 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -516,10 +516,8 @@ export type APIOrganization = { readonly newUserMailingList: string; readonly paidUntil: number; readonly includedUsers: number; - readonly includedStorage: number; // megabytes -}; -export type APIOrganizationStorageInfo = { - readonly usedStorageSpace: number; + readonly includedStorageBytes: number; + readonly usedStorageBytes: number; }; export type APIPricingPlanStatus = { readonly pricingPlan: PricingPlanEnum; diff --git a/test/db/dataStores.csv b/test/db/dataStores.csv index aedd90f19bb..d33e00347b1 100644 --- a/test/db/dataStores.csv +++ b/test/db/dataStores.csv @@ -1,3 +1,3 @@ -name,url,publicUrl,key,isScratch,isDeleted,isConnector,allowsUpload,onlyAllowedOrganization -'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,f,t, -'connect','http://localhost:8000','http://localhost:8000','secret-key',f,f,t,f, +name,url,publicUrl,key,isScratch,isDeleted,isConnector,allowsUpload,onlyAllowedOrganization,reportUsedStorageEnabled +'localhost','http://localhost:9000','http://localhost:9000','something-secure',f,f,f,t,,f +'connect','http://localhost:8000','http://localhost:8000','secret-key',f,f,t,f,,f diff --git a/test/db/organizations.csv b/test/db/organizations.csv index 3f71d9783e4..77dec9ee375 100644 --- a/test/db/organizations.csv +++ b/test/db/organizations.csv @@ -1,3 +1,3 @@ -_id,name,additionalinformation,logoUrl,displayName,_rootFolder,newusermailinglist,overtimemailinglist,enableautoverify,pricingPlan,paidUntil,includedUsers,includedStorage,created,isdeleted -'5ab0c6a674d0af7b003b23ac','Organization_X','lorem ipsum','/assets/images/mpi-logos.svg','Organization_X',570b9f4e4bb848d0885ea917,'','',f,'Custom',,,,,0,'2018-03-20 09:30:31.91+01',f -'6bb0c6a674d0af7b003b23bd','Organization_Y','foo bar','/assets/images/mpi-logos.svg','Organization_Y','570b9f4e4bb848d088a83aef','','',f,'Custom',,,,,0,'2018-03-24 09:30:31.91+01',f +_id,name,additionalinformation,logoUrl,displayName,_rootFolder,newusermailinglist,overtimemailinglist,enableautoverify,pricingPlan,paidUntil,includedUsers,includedStorage,lastStorageScanTime,created,isdeleted +'5ab0c6a674d0af7b003b23ac','Organization_X','lorem ipsum','/assets/images/mpi-logos.svg','Organization_X',570b9f4e4bb848d0885ea917,'','',f,'Custom',,,,,0,'2000-01-01 09:30:31.91+01','2018-03-20 09:30:31.91+01',f +'6bb0c6a674d0af7b003b23bd','Organization_Y','foo bar','/assets/images/mpi-logos.svg','Organization_Y','570b9f4e4bb848d088a83aef','','',f,'Custom',,,,,0,'2000-01-01 09:30:31.91+01','2018-03-24 09:30:31.91+01',f diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index 3a13ad67781..d6ddf5348b9 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -19,7 +19,7 @@ START TRANSACTION; CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(95); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(96); COMMIT TRANSACTION; @@ -169,7 +169,8 @@ CREATE TABLE webknossos.dataStores( isDeleted BOOLEAN NOT NULL DEFAULT false, isConnector BOOLEAN NOT NULL DEFAULT false, allowsUpload BOOLEAN NOT NULL DEFAULT true, - onlyAllowedOrganization CHAR(24) + onlyAllowedOrganization CHAR(24), + reportUsedStorageEnabled BOOLEAN NOT NULL DEFAULT false ); CREATE TABLE webknossos.tracingStores( @@ -291,10 +292,22 @@ CREATE TABLE webknossos.organizations( includedStorage BIGINT DEFAULT NULL, lastTermsOfServiceAcceptanceTime TIMESTAMPTZ, lastTermsOfServiceAcceptanceVersion INT NOT NULL DEFAULT 0, + lastStorageScanTime TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01T00:00:00.000Z', created TIMESTAMPTZ NOT NULL DEFAULT NOW(), isDeleted BOOLEAN NOT NULL DEFAULT false ); +CREATE TABLE webknossos.organization_usedStorage( + _organization CHAR(24) NOT NULL, + _dataStore VARCHAR(256) NOT NULL, + _dataSet CHAR(24) NOT NULL, + layerName VARCHAR(256) NOT NULL, + magOrDirectoryName VARCHAR(256) NOT NULL, + usedStorageBytes BIGINT NOT NULL, + lastUpdated TIMESTAMPTZ, + PRIMARY KEY(_organization, _dataStore, _dataSet, layerName, magOrDirectoryName) +); + CREATE TYPE webknossos.USER_PASSWORDINFO_HASHERS AS ENUM ('SCrypt', 'Empty'); CREATE TABLE webknossos.users( _id CHAR(24) PRIMARY KEY, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala index 86654f76404..e8ece1f09fc 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala @@ -51,6 +51,9 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead object AgglomerateSkeleton { val maxEdges: Int = get[Int]("datastore.agglomerateSkeleton.maxEdges") } + object ReportUsedStorage { + val enabled: Boolean = get[Boolean]("datastore.reportUsedStorage.enabled") + } val children = List(WebKnossos, WatchFileSystem, Cache, Isosurface, Redis, AgglomerateSkeleton) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index e7b1ef855a4..e44e816743d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -19,6 +19,7 @@ import java.io.File import com.scalableminds.webknossos.datastore.storage.AgglomerateFileKey import io.swagger.annotations.{Api, ApiImplicitParam, ApiImplicitParams, ApiOperation, ApiResponse, ApiResponses} import play.api.libs.Files +import scala.concurrent.duration._ import scala.concurrent.ExecutionContext.Implicits.global @@ -30,6 +31,7 @@ class DataSourceController @Inject()( accessTokenService: DataStoreAccessTokenService, binaryDataServiceHolder: BinaryDataServiceHolder, connectomeFileService: ConnectomeFileService, + storageUsageService: DSUsedStorageService, uploadService: UploadService )(implicit bodyParsers: PlayBodyParsers) extends Controller @@ -440,6 +442,26 @@ Expects: } } + @ApiOperation(hidden = true, value = "") + def measureUsedStorage(token: Option[String], + organizationName: String, + datasetName: Option[String] = None): Action[AnyContent] = + Action.async { implicit request => + accessTokenService.validateAccess(UserAccessRequest.administrateDataSources(organizationName), + urlOrHeaderToken(token, request)) { + for { + before <- Fox.successful(System.currentTimeMillis()) + usedStorageInBytes: List[DirectoryStorageReport] <- storageUsageService.measureStorage(organizationName, + datasetName) + after = System.currentTimeMillis() + _ = if (after - before > (10 seconds).toMillis) { + val datasetLabel = datasetName.map(n => s" dataset $n of").getOrElse("") + logger.info(s"Measuring storage for$datasetLabel orga $organizationName took ${after - before} ms.") + } + } yield Ok(Json.toJson(usedStorageInBytes)) + } + } + @ApiOperation(hidden = true, value = "") def reload(token: Option[String], organizationName: String, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala index a62d67094c5..3cd7281c1af 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebKnossosClient.scala @@ -20,7 +20,7 @@ import play.api.libs.ws.WSResponse import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ -case class DataStoreStatus(ok: Boolean, url: String) +case class DataStoreStatus(ok: Boolean, url: String, reportUsedStorageEnabled: Option[Boolean] = None) object DataStoreStatus { implicit val jsonFormat: OFormat[DataStoreStatus] = Json.format[DataStoreStatus] @@ -43,6 +43,7 @@ class DSRemoteWebKnossosClient @Inject()( private val dataStoreKey: String = config.Datastore.key private val dataStoreName: String = config.Datastore.name private val dataStoreUri: String = config.Http.uri + private val reportUsedStorageEnabled: Boolean = config.Datastore.ReportUsedStorage.enabled private val webKnossosUri: String = config.Datastore.WebKnossos.uri @@ -53,7 +54,7 @@ class DSRemoteWebKnossosClient @Inject()( def reportStatus(ok: Boolean): Fox[_] = rpc(s"$webKnossosUri/api/datastores/$dataStoreName/status") .addQueryString("key" -> dataStoreKey) - .patch(DataStoreStatus(ok, dataStoreUri)) + .patch(DataStoreStatus(ok, dataStoreUri, Some(reportUsedStorageEnabled))) def reportDataSource(dataSource: InboxDataSourceLike): Fox[_] = rpc(s"$webKnossosUri/api/datastores/$dataStoreName/datasource") @@ -84,7 +85,7 @@ class DSRemoteWebKnossosClient @Inject()( def reserveDataSourceUpload(info: ReserveUploadInformation, userTokenOpt: Option[String]): Fox[Unit] = for { - userToken <- option2Fox(userTokenOpt) ?~> "validateDataSourceUpload.noUserToken" + userToken <- option2Fox(userTokenOpt) ?~> "reserveUpload.noUserToken" _ <- rpc(s"$webKnossosUri/api/datastores/$dataStoreName/reserveUpload") .addQueryString("key" -> dataStoreKey) .addQueryString("token" -> userToken) @@ -94,6 +95,17 @@ class DSRemoteWebKnossosClient @Inject()( def deleteDataSource(id: DataSourceId): Fox[_] = rpc(s"$webKnossosUri/api/datastores/$dataStoreName/deleteDataset").addQueryString("key" -> dataStoreKey).post(id) + def reportUsedStorage(organizationName: String, + datasetName: Option[String], + storageReportEntries: List[DirectoryStorageReport]): Fox[Unit] = + for { + _ <- rpc(s"$webKnossosUri/api/datastores/$dataStoreName/reportUsedStorage") + .addQueryString("key" -> dataStoreKey) + .addQueryString("organizationName" -> organizationName) + .addQueryStringOptional("datasetName", datasetName) + .post(storageReportEntries) + } yield () + def getJobExportProperties(jobId: String): Fox[JobExportProperties] = rpc(s"$webKnossosUri/api/datastores/$dataStoreName/jobExportProperties") .addQueryString("jobId" -> jobId) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala new file mode 100644 index 00000000000..cd6d325e1f3 --- /dev/null +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala @@ -0,0 +1,88 @@ +package com.scalableminds.webknossos.datastore.services + +import com.scalableminds.util.geometry.Vec3Int +import com.scalableminds.util.io.PathUtils +import com.scalableminds.util.tools.{Fox, FoxImplicits} +import com.scalableminds.webknossos.datastore.DataStoreConfig +import com.typesafe.scalalogging.LazyLogging +import net.liftweb.util.Helpers.tryo +import org.apache.commons.io.FileUtils +import play.api.libs.json.{Json, OFormat} + +import java.nio.file.{Files, Path, Paths} +import javax.inject.Inject +import scala.concurrent.ExecutionContext + +case class DirectoryStorageReport( + organizationName: String, + dataSetName: String, + layerName: String, + magOrDirectoryName: String, + usedStorageBytes: Long +) + +object DirectoryStorageReport { + implicit val jsonFormat: OFormat[DirectoryStorageReport] = Json.format[DirectoryStorageReport] +} + +class DSUsedStorageService @Inject()(config: DataStoreConfig)(implicit ec: ExecutionContext) + extends FoxImplicits + with LazyLogging { + + private val baseDir: Path = Paths.get(config.Datastore.baseFolder) + + private def noSymlinksFilter(p: Path) = !Files.isSymbolicLink(p) + + def measureStorage(organizationName: String, dataSetName: Option[String])( + implicit ec: ExecutionContext): Fox[List[DirectoryStorageReport]] = { + def selectedDatasetFilter(p: Path) = dataSetName.forall(name => p.getFileName.toString == name) + + for { + datasetDirectories <- PathUtils.listDirectories(baseDir.resolve(organizationName), + noSymlinksFilter, + selectedDatasetFilter) ?~> "listdir.failed" + storageReportsNested <- Fox.serialCombined(datasetDirectories)(d => measureStorageForDataSet(organizationName, d)) + } yield storageReportsNested.flatten + } + + def measureStorageForDataSet(organizationName: String, dataSetDirectory: Path): Fox[List[DirectoryStorageReport]] = + for { + layerDirectory <- PathUtils.listDirectories(dataSetDirectory, noSymlinksFilter) ?~> "listdir.failed" + storageReportsNested <- Fox.serialCombined(layerDirectory)(l => + measureStorageForLayerDirectory(organizationName, dataSetDirectory, l)) + } yield storageReportsNested.flatten + + def measureStorageForLayerDirectory(organizationName: String, + dataSetDirectory: Path, + layerDirectory: Path): Fox[List[DirectoryStorageReport]] = + for { + magOrOtherDirectory <- PathUtils.listDirectories(layerDirectory, noSymlinksFilter) ?~> "listdir.failed" + storageReportsNested <- Fox.serialCombined(magOrOtherDirectory)(m => + measureStorageForMagOrOtherDirectory(organizationName, dataSetDirectory, layerDirectory, m)) + } yield storageReportsNested + + def measureStorageForMagOrOtherDirectory(organizationName: String, + dataSetDirectory: Path, + layerDirectory: Path, + magOrOtherDirectory: Path): Fox[DirectoryStorageReport] = + for { + usedStorageBytes <- measureStorage(magOrOtherDirectory) + } yield + DirectoryStorageReport( + organizationName, + dataSetDirectory.getFileName.toString, + layerDirectory.getFileName.toString, + normalizeMagName(magOrOtherDirectory.getFileName.toString), + usedStorageBytes + ) + + private def normalizeMagName(name: String): String = + Vec3Int.fromMagLiteral(name, allowScalar = true) match { + case Some(mag) => mag.toMagLiteral(allowScalar = true) + case None => name + } + + def measureStorage(path: Path)(implicit ec: ExecutionContext): Fox[Long] = + tryo(FileUtils.sizeOfDirectoryAsBigInteger(path.toFile).longValue) + +} diff --git a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes index 4efb8744b76..1089cc609bb 100644 --- a/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes +++ b/webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes @@ -76,6 +76,7 @@ POST /datasets POST /datasets/reserveUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.reserveUpload(token: Option[String]) POST /datasets/finishUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.finishUpload(token: Option[String]) POST /datasets/cancelUpload @com.scalableminds.webknossos.datastore.controllers.DataSourceController.cancelUpload(token: Option[String]) +GET /datasets/measureUsedStorage/:organizationName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.measureUsedStorage(token: Option[String], organizationName: String, dataSetName: Option[String]) GET /datasets/:organizationName/:dataSetName/readInboxDataSourceLike @com.scalableminds.webknossos.datastore.controllers.DataSourceController.read(token: Option[String], organizationName: String, dataSetName: String, returnFormatLike: Boolean ?= true) GET /datasets/:organizationName/:dataSetName/readInboxDataSource @com.scalableminds.webknossos.datastore.controllers.DataSourceController.read(token: Option[String], organizationName: String, dataSetName: String, returnFormatLike: Boolean ?= false) POST /datasets/:organizationName/:dataSetName @com.scalableminds.webknossos.datastore.controllers.DataSourceController.update(token: Option[String], organizationName: String, dataSetName: String) diff --git a/webknossos-datastore/conf/standalone-datastore.conf b/webknossos-datastore/conf/standalone-datastore.conf index 8eaf61bf11e..8f95b2cbfde 100644 --- a/webknossos-datastore/conf/standalone-datastore.conf +++ b/webknossos-datastore/conf/standalone-datastore.conf @@ -62,6 +62,7 @@ datastore { port = 6379 } agglomerateSkeleton.maxEdges = 10000 + reportUsedStorage.enabled = false } slackNotifications {