Merge pull request #81 from saalfeldlab/paintera-datasets

Attempt to add dataset specification as defined in #61

README.md

@@ -86,5 +86,39 @@ Usage: Paintera [-h] [--height=HEIGHT] [--width=WIDTH]
 | `R` | Clear mesh caches and refresh meshes (if current source is label source) |
 | `L` | Lock last selected segment (if label source) |
 
+## Data
+
+In [#61](https://github.com/saalfeldlab/paintera/issues/61) we introduced a specification for the data format that Paintera can load through the opener dialog (`Ctrl O`).
+These restrictions hold only for the graphical user interface. If desired, callers can
+ - add arbitrary data sets programatically, or
+ - through the `attributes.json` project file if an appropriate gson deserializer is supplied.
+
+### Raw
+Accept any of these:
+ 1. any regular (i.e. default mode) three-dimensional N5 dataset that is integer or float. Optional attributes are `"resolution": [x,y,z]` and `"offset": [x,y,z]`.
+ 2. any multiscale N5 group that has `"multiScale" : true` attribute and contains three-dimensional multi-scale datasets `s0` ... `sN`. Optional attributes are `"resolution": [x,y,z]` and `"offset: [x,y,z]"`. In addition to the requirements from (1), all `s1` ... `sN` datasets must contain `"downsamplingFactors": [x,y,z]` entry (`s0` is exempt, will default to `[1.0, 1.0, 1.0]`). All datasets must have same type. Optional attributes from (1) will be ignored.
+ 3. (preferred) any N5 group with attribute `"painteraData : {"type" : "raw"}` and a dataset/group `data` that conforms with (1) or (2).
+
+### Labels
+Accept any of these:
+ 1. any regular (i.e. default mode) integer or varlength `LabelMultisetType` (`"isLabelMultiset": true`) three-dimensional N5 dataset. Optional attributes are `"resolution": [x,y,z]`, `"offset": [x,y,z]`, `"maxId": <id>`. If `"maxId"` is not specified, it is determined at start-up and added.
+ 2. any multiscale N5 group that has `"multiScale" : true` attribute and contains three-dimensional multi-scale datasets `s0` ... `sN`. Optional attributes are `"resolution": [x,y,z]`, `"offset": [x,y,z]`, `"maxId": <id>`. If `"maxId"` is not specified, it is determined at start-up and added (this can be expensive). In addition to the requirements from (1), all `s1` ... `sN` datasets must contain `"downsamplingFactors": [x,y,z]` entry (`s0` is exempt, will default to `[1.0, 1.0, 1.0]`). All datasets must have same type. Optional attributes from (1) will be ignored.
+ 3. (preferred) any N5 group with attribute `"painteraData : {"type" : "label"}` and a dataset/group `data` that conforms with (1) or (2). Optional sub-groups are:
+   - `fragment-segment-assignment` -- Dataset to store fragment-segment lookup table. Can be empty or will be initialized empty if it does not exist.
+   - `unique-label-lists`          -- Multiscale varlength dataset with same 'dimensions'/'blockSize' and as dataset(s) in `data`. Holds unique block lists from which relevant blocks for specific ids are retrieved.
+
+#### Things to consider for labels:
+ - make `"maxId"` attribute mandatory because `IdService` needs it and would require to scan whole dataset (can be huge)
+ - Efficient mesh generation
+  - only possible if
+    - option (3) and `unique-label-lists` exists, or
+    - `LabelMultisetType` dataset (currently, `VolatileLabelMultisetArray` holds a set of contained labels).
+  - If none of these are true, ask user if
+    - unique label lists should be generated from highest resolution to lowest resolution (slow, would need to be generated once and cached or saved as `unique-label-lists` if option (3), probably useful for looking at small datasets), or
+    - generate unique label lists on the fly from lowest resolution to highest resolution (fast, only do the work that's currently necessary, but potentially incomplete), or
+    - 3D support should be disabled for that source.
+ - If (1) or (2), fragment-segment-assignment cannot be committed back to source (only stored as actions in project's `attributes.json`). There should be an option to export fragment-segment-assignment as N5 dataset. That way, users can choose to update their source in order to conform with (3) and not lose their work on fragment-segment assignments
+ - (3) leaves room for adding related data in the future, e.g. annotations
+
 
 

src/main/java/org/janelia/saalfeldlab/paintera/N5Helpers.java

@@ -113,6 +113,12 @@ public class N5Helpers
 
 	public static final String MAX_NUM_ENTRIES_KEY = "maxNumEntries";
 
+	public static final String PAINTERA_DATA_KEY = "painteraData";
+
+	public static final String PAINTERA_DATA_DATASET = "data";
+
+	public static final String PAINTERA_FRAGMENT_SEGMENT_ASSIGNMENT_DATASTE = "fragment-segment-assignment";
+
 	private static final Logger LOG = LoggerFactory.getLogger( MethodHandles.lookup().lookupClass() );
 
 	public static boolean isIntegerType( final DataType type )
@@ -133,6 +139,11 @@ public static boolean isIntegerType( final DataType type )
 		}
 	}
 
+	public static boolean isPainteraDataset( final N5Reader n5, final String group ) throws IOException
+	{
+		return n5.exists( group ) && n5.listAttributes( group ).containsKey( PAINTERA_DATA_KEY );
+	}
+
 	public static boolean isMultiScale( final N5Reader n5, final String dataset ) throws IOException
 	{
 		/* based on attribute */
@@ -245,6 +256,14 @@ public static String[] listAndSortScaleDatasets( final N5Reader n5, final String
 		return scaleDirs;
 	}
 
+	public static DataType getDataType( final N5Reader n5, final String group ) throws IOException
+	{
+		LOG.warn( "Getting data type for group/dataset {}", group );
+		if ( isPainteraDataset( n5, group ) ) { return getDataType( n5, group + "/" + PAINTERA_DATA_DATASET ); }
+		if ( isMultiScale( n5, group ) ) { return getDataType( n5, getFinestLevel( n5, group ) ); }
+		return n5.getDatasetAttributes( group ).getDataType();
+	}
+
 	public static void sortScaleDatasets( final String[] scaleDatasets )
 	{
 		Arrays.sort( scaleDatasets, ( f1, f2 ) -> {
@@ -319,7 +338,14 @@ public static void discoverSubdirectories(
 	{
 		try
 		{
-			if ( n5.datasetExists( pathName ) )
+			if ( isPainteraDataset( n5, pathName ) )
+			{
+				synchronized ( datasets )
+				{
+					datasets.add( pathName );
+				}
+			}
+			else if ( n5.datasetExists( pathName ) )
 			{
 				synchronized ( datasets )
 				{
@@ -328,8 +354,10 @@ public static void discoverSubdirectories(
 			}
 			else
 			{
+
 				String[] groups = null;
 				/* based on attribute */
+
 				boolean isMipmapGroup = Optional.ofNullable( n5.getAttribute( pathName, MULTI_SCALE_KEY, Boolean.class ) ).orElse( false );
 
 				/* based on groupd content (the old way) */
@@ -802,12 +830,19 @@ public static AffineTransform3D considerDownsampling(
 		return transform.concatenate( new Translation3D( shift ) );
 	}
 
-	public static FragmentSegmentAssignmentState assignments( final N5Writer writer, final String ds ) throws IOException
+	public static FragmentSegmentAssignmentState assignments( final N5Writer writer, final String group ) throws IOException
 	{
-		final String dataset = ds + ".fragment-segment-assignment";
+
+		if ( !isPainteraDataset( writer, group ) ) { return new FragmentSegmentAssignmentOnlyLocal(
+				TLongLongHashMap::new,
+				( ks, vs ) -> {
+					throw new UnableToPersist( "Persisting assignments not supported for non Paintera group/dataset " + group );
+				} ); }
+
+		final String dataset = group + "/" + PAINTERA_FRAGMENT_SEGMENT_ASSIGNMENT_DATASTE;
 
 		final Persister persister = ( keys, values ) -> {
-			// TODO handle zero length assignments?
+			// TODO how to handle zero length assignments?
 			if ( keys.length == 0 ) { throw new UnableToPersist( "Zero length data, will not persist fragment-segment-assignment." ); }
 			try
 			{
@@ -866,26 +901,25 @@ public static IdService idService( final N5Writer n5, final String dataset ) thr
 		final long actualMaxId;
 		if ( maxId == null )
 		{
-			final boolean isMultiscale = isMultiScale( n5, dataset );
-			final String dsPath = dataset;
-			// isMultiscale ? Paths.get( dataset, getCoarsestLevel( n5, dataset
-			// ) ).toString() : dataset;
-			final DatasetAttributes attributes = n5.getDatasetAttributes( dsPath );
-			LOG.debug( "is multiscale? {} dsPath={}", isMultiscale, dsPath );
-			if ( isLabelMultisetType( n5, dsPath, isMultiscale ) )
+			final String ds = isPainteraDataset( n5, dataset ) ? dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : dataset;
+			final boolean isMultiscale = isMultiScale( n5, ds );
+			final DatasetAttributes attributes = n5.getDatasetAttributes( ds );
+			LOG.debug( "is multiscale? {} dsPath={}", isMultiscale, ds );
+			if ( isLabelMultisetType( n5, ds, isMultiscale ) )
 			{
 				LOG.debug( "Getting id service for label multisets" );
-				actualMaxId = maxIdLabelMultiset( n5, dsPath );
+				actualMaxId = maxIdLabelMultiset( n5, ds );
 				LOG.debug( "Got max id={}", actualMaxId );
 			}
 			else if ( isIntegerType( attributes.getDataType() ) )
 			{
-				actualMaxId = maxId( n5, dsPath );
+				actualMaxId = maxId( n5, ds );
 			}
 			else
 			{
 				return null;
 			}
+			n5.setAttribute( dataset, "maxId", actualMaxId );
 		}
 		else
 		{
@@ -959,6 +993,7 @@ public static String getFinestLevel(
 			final N5Reader n5,
 			final String dataset ) throws IOException
 	{
+		LOG.warn( "Getting finest level for dataset {}", dataset );
 		final String[] scaleDirs = listAndSortScaleDatasets( n5, dataset );
 		return Paths.get( dataset, scaleDirs[ 0 ] ).toString();
 	}
@@ -973,6 +1008,7 @@ public static String getCoarsestLevel(
 
 	public static double[] getDoubleArrayAttribute( final N5Reader n5, final String dataset, final String key, final double... fallBack ) throws IOException
 	{
+		if ( isPainteraDataset( n5, dataset ) ) { return getDoubleArrayAttribute( n5, dataset + "/" + PAINTERA_DATA_DATASET, key, fallBack ); }
 		return Optional.ofNullable( n5.getAttribute( dataset, key, double[].class ) ).orElse( fallBack );
 	}
 

src/main/java/org/janelia/saalfeldlab/paintera/Paintera.java

@@ -296,11 +296,11 @@ public void startImpl( final Stage stage ) throws Exception
 			}
 			catch ( final CannotPersist e1 )
 			{
-				LOG.error( "Unable to persist canvas {}", e1 );
+				LOG.error( "Unable to persist canvas: {}", e1.getMessage() );
 			}
 			catch ( final UnableToPersist e1 )
 			{
-				LOG.error( "Unable to persist fragment-segment-assignment {}", e1 );
+				LOG.error( "Unable to persist fragment-segment-assignment: {}", e1.getMessage() );
 			}
 		},
 				e -> keyTracker.areOnlyTheseKeysDown( KeyCode.CONTROL, KeyCode.C ) ).installInto( paneWithStatus.getPane() );
@@ -392,8 +392,9 @@ private static < D extends NativeType< D >, T extends Volatile< D > & NativeType
 				final String[] split = identifier.replaceFirst( "file://", "" ).split( ":" );
 				final N5Writer n5 = N5Helpers.n5Writer( split[ 0 ], 64, 64, 64 );
 				final String dataset = split[ 1 ];
-				final double[] resolution = Optional.ofNullable( n5.getAttribute( dataset, "resolution", double[].class ) ).orElse( new double[] { 1.0, 1.0, 1.0 } );
-				final double[] offset = Optional.ofNullable( n5.getAttribute( dataset, "offset", double[].class ) ).orElse( new double[] { 0.0, 0.0, 0.0 } );
+				LOG.warn( "Adding label dataset={} dataset={}", split[ 0 ], dataset );
+				final double[] resolution = N5Helpers.getResolution( n5, dataset );
+				final double[] offset = N5Helpers.getOffset( n5, dataset );
 				final AffineTransform3D transform = N5Helpers.fromResolutionAndOffset( resolution, offset );
 				final Supplier< String > nextCanvasDir = Masks.canvasTmpDirDirectorySupplier( projectDirectory );
 				final String name = N5Helpers.lastSegmentOfDatasetPath( dataset );

src/main/java/org/janelia/saalfeldlab/paintera/data/n5/CommitCanvasN5.java

@@ -69,15 +69,14 @@ public void accept( final CachedCellImg< UnsignedLongType, ? > canvas, final lon
 	{
 		try
 		{
-			final boolean isMultiscale = !n5.datasetExists( dataset );
+			final String dataset = N5Helpers.isPainteraDataset( n5, this.dataset ) ? this.dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : this.dataset;
+			final boolean isMultiscale = N5Helpers.isMultiScale( n5, dataset );
 
 			final CellGrid canvasGrid = canvas.getCellGrid();
 
 			final String highestResolutionDataset = isMultiscale ? Paths.get( dataset, N5Helpers.listAndSortScaleDatasets( n5, dataset )[ 0 ] ).toString() : dataset;
 
-			if ( !Optional.ofNullable( n5.getAttribute( highestResolutionDataset, N5Helpers.LABEL_MULTISETTYPE_KEY, Boolean.class ) ).orElse( false ) ) {
-				throw new RuntimeException( "Only label multiset type accepted currently!" );
-			}
+			if ( !Optional.ofNullable( n5.getAttribute( highestResolutionDataset, N5Helpers.LABEL_MULTISETTYPE_KEY, Boolean.class ) ).orElse( false ) ) { throw new RuntimeException( "Only label multiset type accepted currently!" ); }
 
 			final DatasetAttributes highestResolutionAttributes = n5.getDatasetAttributes( highestResolutionDataset );
 			final CellGrid highestResolutionGrid = new CellGrid( highestResolutionAttributes.getDimensions(), highestResolutionAttributes.getBlockSize() );

src/main/java/org/janelia/saalfeldlab/paintera/data/n5/N5DataSource.java

@@ -81,7 +81,7 @@ private static < T extends NativeType< T > > Function< Interpolation, Interpolat
 	{
 		return N5Helpers.isLabelMultisetType( n5, dataset )
 				? i -> new NearestNeighborInterpolatorFactory<>()
-						: ( Function ) realTypeInterpolation();
+				: ( Function ) realTypeInterpolation();
 	}
 
 	private static < T extends RealType< T > > Function< Interpolation, InterpolatorFactory< T, RandomAccessible< T > > > realTypeInterpolation()
@@ -91,27 +91,28 @@ private static < T extends RealType< T > > Function< Interpolation, Interpolator
 
 	@SuppressWarnings( { "unchecked", "rawtypes" } )
 	private static < D extends NativeType< D >, T extends Volatile< D > & NativeType< T > >
-	Triple< RandomAccessibleInterval< D >[], RandomAccessibleInterval< T >[], AffineTransform3D[] > getData(
-			final N5Reader reader,
-			final String dataset,
-			final AffineTransform3D transform,
-			final SharedQueue sharedQueue,
-			final int priority ) throws IOException
+			Triple< RandomAccessibleInterval< D >[], RandomAccessibleInterval< T >[], AffineTransform3D[] > getData(
+					final N5Reader reader,
+					final String dataset,
+					final AffineTransform3D transform,
+					final SharedQueue sharedQueue,
+					final int priority ) throws IOException
 	{
+		if ( N5Helpers.isPainteraDataset( reader, dataset ) ) { return getData( reader, dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET, transform, sharedQueue, priority ); }
 		final boolean isMultiscale = N5Helpers.isMultiScale( reader, dataset );
 		final boolean isLabelMultiset = N5Helpers.isLabelMultisetType( reader, dataset, isMultiscale );
 
 		if ( isLabelMultiset )
 		{
 			return isMultiscale
 					? ( Triple ) N5Helpers.openLabelMultisetMultiscale( reader, dataset, transform, sharedQueue, priority )
-							: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openLabelMutliset( reader, dataset, transform, sharedQueue, priority ) );
+					: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openLabelMutliset( reader, dataset, transform, sharedQueue, priority ) );
 		}
 		else
 		{
 			return isMultiscale
 					? ( Triple ) N5Helpers.openRawMultiscale( reader, dataset, transform, sharedQueue, priority )
-							: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openRaw( reader, dataset, transform, sharedQueue, priority ) );
+					: ( Triple ) N5Helpers.asArrayTriple( N5Helpers.openRaw( reader, dataset, transform, sharedQueue, priority ) );
 		}
 	}
 }

src/main/java/org/janelia/saalfeldlab/paintera/ui/opendialog/GenericBackendDialogN5.java

@@ -219,21 +219,21 @@ public GenericBackendDialogN5(
 		dataset.set( "" );
 	}
 
-	public void updateDatasetInfo( final String dataset, final DatasetInfo info )
+	public void updateDatasetInfo( final String group, final DatasetInfo info )
 	{
 
-		LOG.debug( "Updating dataset info for dataset {}", dataset );
+		LOG.debug( "Updating dataset info for dataset {}", group );
 		try
 		{
 			final N5Reader n5 = this.n5.get();
-			final String ds = N5Helpers.isMultiScale( n5, dataset ) ? N5Helpers.getFinestLevel( n5, dataset ) : dataset;
-			LOG.debug( "Got dataset={}, ds={}", dataset, ds );
-
-			setResolution( N5Helpers.getResolution( n5, dataset ) );
-			setOffset( N5Helpers.getOffset( n5, dataset ) );
-			final DatasetAttributes dsAttrs = n5.getDatasetAttributes( ds );
-			this.datasetInfo.minProperty().set( Optional.ofNullable( n5.getAttribute( dataset, MIN_KEY, Double.class ) ).orElse( N5Helpers.minForType( dsAttrs.getDataType() ) ) );
-			this.datasetInfo.maxProperty().set( Optional.ofNullable( n5.getAttribute( dataset, MAX_KEY, Double.class ) ).orElse( N5Helpers.maxForType( dsAttrs.getDataType() ) ) );
+
+			setResolution( N5Helpers.getResolution( n5, group ) );
+			setOffset( N5Helpers.getOffset( n5, group ) );
+
+			final DataType dataType = N5Helpers.getDataType( n5, group );
+
+			this.datasetInfo.minProperty().set( Optional.ofNullable( n5.getAttribute( group, MIN_KEY, Double.class ) ).orElse( N5Helpers.minForType( dataType ) ) );
+			this.datasetInfo.maxProperty().set( Optional.ofNullable( n5.getAttribute( group, MAX_KEY, Double.class ) ).orElse( N5Helpers.maxForType( dataType ) ) );
 		}
 		catch ( final IOException e )
 		{
@@ -290,22 +290,25 @@ public IdService idService()
 			final String dataset = this.dataset.get();
 
 			final Long maxId = n5.getAttribute( dataset, "maxId", Long.class );
+			final boolean isPainteraData = N5Helpers.isPainteraDataset( n5, dataset );
 			final long actualMaxId;
 			if ( maxId == null )
 			{
+				final String ds = isPainteraData ? dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : dataset;
 				if ( isLabelMultisetType() )
 				{
 					LOG.debug( "Getting id service for label multisets" );
-					actualMaxId = maxIdLabelMultiset( n5, dataset );
+					actualMaxId = maxIdLabelMultiset( n5, ds );
 				}
 				else if ( isIntegerType() )
 				{
-					actualMaxId = maxId( n5, dataset );
+					actualMaxId = maxId( n5, ds );
 				}
 				else
 				{
 					return null;
 				}
+				n5.setAttribute( dataset, "maxId", actualMaxId );
 			}
 			else
 			{
@@ -490,7 +493,12 @@ public boolean isLabelType() throws Exception
 
 	public boolean isLabelMultisetType() throws Exception
 	{
-		final Boolean attribute = getAttribute( LABEL_MULTISETTYPE_KEY, Boolean.class );
+		final N5Writer n5 = this.n5.get();
+		final String dataset = this.dataset.get();
+		final Boolean attribute = n5.getAttribute(
+				N5Helpers.isPainteraDataset( n5, dataset ) ? dataset + "/" + N5Helpers.PAINTERA_DATA_DATASET : dataset,
+				N5Helpers.LABEL_MULTISETTYPE_KEY,
+				Boolean.class );
 		LOG.debug( "Getting label multiset attribute: {}", attribute );
 		return Optional.ofNullable( attribute ).orElse( false );
 	}
@@ -535,28 +543,6 @@ public boolean isIntegerType() throws Exception
 		return new CommitCanvasN5( writer, dataset );
 	}
 
-	public < T > T getAttribute( final String key, final Class< T > clazz ) throws IOException
-	{
-		final N5Reader n5 = this.n5.get();
-		final String ds = this.dataset.get();
-
-		if ( n5.datasetExists( ds ) )
-		{
-			LOG.debug( "Getting attributes for {} and {}", n5, ds );
-			return n5.getAttribute( ds, key, clazz );
-		}
-
-		final String[] scaleDirs = N5Helpers.listAndSortScaleDatasets( n5, ds );
-
-		if ( scaleDirs.length > 0 )
-		{
-			LOG.warn( "Getting attributes for mipmap dataset: {} and {}", n5, scaleDirs[ 0 ] );
-			return n5.getAttribute( Paths.get( ds, scaleDirs[ 0 ] ).toString(), key, clazz );
-		}
-
-		throw new RuntimeException( String.format( "Cannot read dataset attributes for group %s and dataset %s.", n5, ds ) );
-	}
-
 	public ExecutorService propagationExecutor()
 	{
 		return this.propagationExecutor;