Skip to content

Commit

Permalink
feat: N5Factory add s3ClientConfiguration
Browse files Browse the repository at this point in the history 
* chore: bump n5-zarr
* chore: bump n5-aws-s3 4.2.0
* style: code style
  • Loading branch information
@bogovicj
bogovicj authored Jun 5, 2024
1 parent 75a5866 commit b98e9f2
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 39 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,11 @@
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

<n5.version>3.2.0</n5.version>
<n5-aws-s3.version>4.1.2</n5-aws-s3.version>
<n5-aws-s3.version>4.2.0</n5-aws-s3.version>
<n5-hdf5.version>2.2.0</n5-hdf5.version>
<n5-imglib2.version>7.0.0</n5-imglib2.version>
<n5-google-cloud.version>4.1.0</n5-google-cloud.version>
<n5-zarr.version>1.3.1</n5-zarr.version>
<n5-zarr.version>1.3.4</n5-zarr.version>

<jackson-jq.version>1.0.0-preview.20191208</jackson-jq.version>
<alphanumeric-comparator.version>1.4.1</alphanumeric-comparator.version>
Expand Down
73 changes: 42 additions & 31 deletions src/main/java/org/janelia/saalfeldlab/n5/universe/N5Factory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,19 @@
*/
package org.janelia.saalfeldlab.n5.universe;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.google.cloud.storage.Storage;
import com.google.gson.GsonBuilder;
import net.imglib2.util.Pair;
import net.imglib2.util.ValuePair;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;

import org.apache.commons.lang3.function.TriFunction;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudStorageURI;
import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils;
Expand All @@ -55,17 +61,15 @@
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader;
import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueWriter;

import javax.annotation.Nullable;
import java.io.File;
import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.google.cloud.storage.Storage;
import com.google.gson.GsonBuilder;

import net.imglib2.util.Pair;
import net.imglib2.util.ValuePair;

/**
* Factory for various N5 readers and writers. Implementation specific
Expand Down Expand Up @@ -94,6 +98,7 @@ public class N5Factory implements Serializable {
private String googleCloudProjectId = null;
private String s3Region = null;
private AWSCredentials s3Credentials = null;
private ClientConfiguration s3ClientConfiguration = null;
private boolean s3Anonymous = true;
private String s3Endpoint;

Expand Down Expand Up @@ -182,6 +187,12 @@ public N5Factory s3UseCredentials(final AWSCredentials credentials) {
return this;
}

public N5Factory s3ClientConfiguration(final ClientConfiguration clientConfiguration) {

this.s3ClientConfiguration = clientConfiguration;
return this;
}

@Deprecated
public N5Factory s3RetryWithCredentials() {

Expand All @@ -203,7 +214,7 @@ public N5Factory s3Region(final String s3Region) {
AmazonS3 createS3(final String uri) {

try {
return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3Region);
return AmazonS3Utils.createS3(uri, s3Endpoint, AmazonS3Utils.getS3Credentials(s3Credentials, s3Anonymous), s3ClientConfiguration, s3Region);
} catch (final Throwable e) {
throw new N5Exception("Could not create s3 client from uri: " + uri, e);
}
Expand All @@ -228,7 +239,7 @@ KeyValueAccess getKeyValueAccess(final URI uri) {
* backend take into account reasonable defaults when possible.
* Here we test from most to least restrictive.
* See the Javadoc for more details. */
for (KeyValueAccessBackend backend : KeyValueAccessBackend.values()) {
for (final KeyValueAccessBackend backend : KeyValueAccessBackend.values()) {
final KeyValueAccess kva = backend.apply(uri, this);
if (kva != null)
return kva;
Expand Down Expand Up @@ -352,10 +363,10 @@ public N5Reader openReader(final String uri) {
private N5Reader openReader(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, String containerPath) {

if (storage == null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
try {
return openReader(format, access, containerPath);
} catch (Throwable e) {
} catch (final Throwable e) {
}
}
throw new N5Exception("Unable to open " + containerPath + " as N5Reader");
Expand Down Expand Up @@ -480,10 +491,10 @@ public N5Writer openWriter(final String uri) {
private N5Writer openWriter(@Nullable final StorageFormat storage, @Nullable final KeyValueAccess access, final String containerPath) {

if (storage == null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
try {
return openWriter(format, access, containerPath);
} catch (Throwable ignored) {
} catch (final Throwable ignored) {
}
}
throw new N5Exception("Unable to open " + containerPath + " as N5Writer");
Expand Down Expand Up @@ -511,7 +522,7 @@ private <T extends N5Reader> T openN5ContainerWithStorageFormat(
try {
final URI asUri = StorageFormat.parseUri(uri).getB();
return openWithFormat.apply(format, asUri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception("Cannot create N5 Container (" + format + ") at " + uri, e);
}
}
Expand Down Expand Up @@ -547,7 +558,7 @@ private <T extends N5Reader> T openN5Container(
final Pair<StorageFormat, URI> storageAndUri;
try {
storageAndUri = StorageFormat.parseUri(containerUri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception("Unable to open " + containerUri + " as N5 Container", e);
}
final StorageFormat format = storageAndUri.getA();
Expand Down Expand Up @@ -631,7 +642,7 @@ public enum StorageFormat {

public static StorageFormat guessStorageFromUri(URI uri) {

for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
if (format.uriTest.test(uri))
return format;
}
Expand All @@ -656,7 +667,7 @@ public static Pair<StorageFormat, String> getStorageFromNestedScheme(String uri)
final String storageFormatScheme = storageSchemeMatcher.group(STORAGE_SCHEME_GROUP);
final String uriGroup = storageSchemeMatcher.group(URI_GROUP);
if (storageFormatScheme != null) {
for (StorageFormat format : StorageFormat.values()) {
for (final StorageFormat format : StorageFormat.values()) {
if (format.schemePattern.asPredicate().test(storageFormatScheme))
return new ValuePair<>(format, uriGroup);
}
Expand Down Expand Up @@ -690,15 +701,15 @@ public static N5Reader createReader(String containerUri) {
private static URI parseUriFromString(String uri) {
try {
return URI.create(uri);
} catch (Throwable ignore) {}
} catch (final Throwable ignore) {}

try {
return Paths.get(uri).toUri();
} catch (Throwable ignore) {}
} catch (final Throwable ignore) {}

try {
return N5URI.encodeAsUri(uri);
} catch (URISyntaxException e) {
} catch (final URISyntaxException e) {
throw new N5Exception(e);
}
}
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/janelia/saalfeldlab/n5/universe/N5TreeNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ public N5TreeNode addPath( final String path, Function<String, N5TreeNode> const

// get the appropriate child along the path if it exists, otherwise add it
N5TreeNode child = null;
Stream<N5TreeNode> cs = children.stream().filter( n -> n.getNodeName().equals(childName));
Optional<N5TreeNode> copt = cs.findFirst();
final Stream<N5TreeNode> cs = children.stream().filter( n -> n.getNodeName().equals(childName));
final Optional<N5TreeNode> copt = cs.findFirst();
if( copt.isPresent() )
child = copt.get();
else {
Expand Down Expand Up @@ -199,8 +199,8 @@ public boolean structureEquals( N5TreeNode other )
return false;

boolean childrenEqual = true;
for( N5TreeNode c : childrenList()) {
Optional<N5TreeNode> otherChildOpt = other.childrenList().stream()
for( final N5TreeNode c : childrenList()) {
final Optional<N5TreeNode> otherChildOpt = other.childrenList().stream()
.filter( x -> x.getNodeName().equals( c.getNodeName()))
.findFirst();

Expand All @@ -221,9 +221,9 @@ public String printRecursive() {

private static String printRecursiveHelper(N5TreeNode node, String prefix) {

StringBuffer out = new StringBuffer();
final StringBuffer out = new StringBuffer();
out.append(prefix + node.path + "\n");
for (N5TreeNode c : node.childrenList()) {
for (final N5TreeNode c : node.childrenList()) {
System.out.println(c.path);
out.append(printRecursiveHelper(c, prefix + " "));
}
Expand Down

0 comments on commit b98e9f2

Please sign in to comment.