diff --git a/gcloud-java-contrib/gcloud-java-nio/pom.xml b/gcloud-java-contrib/gcloud-java-nio/pom.xml new file mode 100644 index 000000000000..69d3c5157c96 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + com.google.gcloud + gcloud-java-nio + jar + GCloud Java NIO + + FileSystemProvider for Java NIO to access GCS transparently. + + + com.google.gcloud + gcloud-java-contrib + 0.1.4-SNAPSHOT + + + nio + + + + ${project.groupId} + gcloud-java + ${project.version} + + + com.google.guava + guava + 19.0 + + + com.google.code.findbugs + jsr305 + 2.0.1 + + + javax.inject + javax.inject + 1 + + + com.google.auto.service + auto-service + 1.0-rc2 + provided + + + com.google.auto.value + auto-value + 1.1 + provided + + + com.google.appengine.tools + appengine-gcs-client + 0.5 + + + junit + junit + 4.12 + test + + + com.google.guava + guava-testlib + 19.0 + test + + + com.google.truth + truth + 0.27 + test + + + org.mockito + mockito-core + 1.9.5 + + + com.google.appengine + appengine-testing + 1.9.30 + test + + + com.google.appengine + appengine-api-stubs + 1.9.30 + test + + + com.google.appengine + appengine-local-endpoints + 1.9.30 + test + + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + UTF-8 + -Xlint:unchecked + + + + maven-jar-plugin + 2.6 + + + true + true + + true + true + + + ${project.artifactId} + ${project.groupId} + ${project.version} + ${buildNumber} + + + + + + + diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java new file mode 100644 index 000000000000..e939edfb5a19 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfiguration.java @@ -0,0 +1,148 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; + +import java.util.Map; + +/** Configuration class for {@link CloudStorageFileSystem#forBucket} */ +@AutoValue +public abstract class CloudStorageConfiguration { + + /** Returns the path of the current working directory. Defaults to the root directory. */ + public abstract String workingDirectory(); + + /** + * Returns {@code true} if we shouldn't throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b}. + */ + public abstract boolean permitEmptyPathComponents(); + + /** + * Returns {@code true} if '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public abstract boolean stripPrefixSlash(); + + /** Return {@code true} if paths with a trailing slash should be treated as fake directories. */ + public abstract boolean usePseudoDirectories(); + + /** Returns the block size (in bytes) used when talking to the GCS HTTP server. */ + public abstract int blockSize(); + + /** + * Creates a new builder, initialized with the following settings: + * + *

+ */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for {@link CloudStorageConfiguration}. */ + public static final class Builder { + + private String workingDirectory = UnixPath.ROOT; + private boolean permitEmptyPathComponents = false; + private boolean stripPrefixSlash = true; + private boolean usePseudoDirectories = true; + private int blockSize = CloudStorageFileSystem.BLOCK_SIZE_DEFAULT; + + /** + * Changes the current working directory for a new filesystem. This cannot be changed once it's + * been set. You'll need to simply create another filesystem object. + * + * @throws IllegalArgumentException if {@code path} is not absolute. + */ + public Builder workingDirectory(String path) { + checkArgument(UnixPath.getPath(false, path).isAbsolute(), "not absolute: %s", path); + workingDirectory = path; + return this; + } + + /** + * Configures whether or not we should throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b} + */ + public Builder permitEmptyPathComponents(boolean value) { + permitEmptyPathComponents = value; + return this; + } + + /** + * Configures if the '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public Builder stripPrefixSlash(boolean value) { + stripPrefixSlash = value; + return this; + } + + /** Configures if paths with a trailing slash should be treated as fake directories. */ + public Builder usePseudoDirectories(boolean value) { + usePseudoDirectories = value; + return this; + } + + /** + * Sets the block size in bytes that should be used for each HTTP request to the API. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public Builder blockSize(int value) { + blockSize = value; + return this; + } + + /** Creates a new instance, but does not destroy the builder. */ + public CloudStorageConfiguration build() { + return new AutoValue_CloudStorageConfiguration( + workingDirectory, + permitEmptyPathComponents, + stripPrefixSlash, + usePseudoDirectories, + blockSize); + } + + Builder() {} + } + + static final CloudStorageConfiguration DEFAULT = builder().build(); + + static CloudStorageConfiguration fromMap(Map env) { + Builder builder = builder(); + for (Map.Entry entry : env.entrySet()) { + switch (entry.getKey()) { + case "workingDirectory": + builder.workingDirectory((String) entry.getValue()); + break; + case "permitEmptyPathComponents": + builder.permitEmptyPathComponents((Boolean) entry.getValue()); + break; + case "stripPrefixSlash": + builder.stripPrefixSlash((Boolean) entry.getValue()); + break; + case "usePseudoDirectories": + builder.usePseudoDirectories((Boolean) entry.getValue()); + break; + case "blockSize": + builder.blockSize((Integer) entry.getValue()); + break; + default: + throw new IllegalArgumentException(entry.getKey()); + } + } + return builder.build(); + } + + CloudStorageConfiguration() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java new file mode 100644 index 000000000000..5a4d4bc7c28c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeView.java @@ -0,0 +1,78 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Verify.verifyNotNull; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.common.base.MoreObjects; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Objects; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** Metadata view for a Google Cloud Storage object. */ +@Immutable +public final class CloudStorageFileAttributeView implements BasicFileAttributeView { + + private final CloudStorageFileSystemProvider provider; + private final CloudStoragePath path; + + CloudStorageFileAttributeView(CloudStorageFileSystemProvider provider, CloudStoragePath path) { + this.provider = verifyNotNull(provider); + this.path = verifyNotNull(path); + } + + /** Returns {@value CloudStorageFileSystem#GCS_VIEW} */ + @Override + public String name() { + return CloudStorageFileSystem.GCS_VIEW; + } + + @Override + public CloudStorageFileAttributes readAttributes() throws IOException { + if (path.seemsLikeADirectory() + && path.getFileSystem().config().usePseudoDirectories()) { + return CloudStoragePseudoDirectoryAttributes.SINGLETON_INSTANCE; + } + GcsFileMetadata metadata = provider.getGcsService().getMetadata(path.getGcsFilename()); + if (metadata == null) { + throw new NoSuchFileException(path.toUri().toString()); + } + return new CloudStorageObjectAttributes(metadata); + } + + /** + * This feature is not supported, since Cloud Storage objects are immutable. + * + * @throws UnsupportedOperationException + */ + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) { + throw new CloudStorageObjectImmutableException(); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof CloudStorageFileAttributeView + && Objects.equals(provider, ((CloudStorageFileAttributeView) other).provider) + && Objects.equals(path, ((CloudStorageFileAttributeView) other).path); + } + + @Override + public int hashCode() { + return Objects.hash(provider, path); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("provider", provider) + .add("path", path) + .toString(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java new file mode 100644 index 000000000000..774a5499431d --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributes.java @@ -0,0 +1,59 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.BasicFileAttributes; + +/** Interface for attributes on a cloud storage file or pseudo-directory. */ +public interface CloudStorageFileAttributes extends BasicFileAttributes { + + /** + * Returns the HTTP etag hash for this object. + * + * @see "https://developers.google.com/storage/docs/hashes-etags" + */ + Optional etag(); + + /** + * Returns the mime type (e.g. text/plain) if it was set for this object. + * + * @see "http://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types" + */ + Optional mimeType(); + + /** + * Returns the ACL value on this Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + Optional acl(); + + /** + * Returns the {@code Cache-Control} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + Optional cacheControl(); + + /** + * Returns the {@code Content-Encoding} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + Optional contentEncoding(); + + /** + * Returns the {@code Content-Disposition} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + Optional contentDisposition(); + + /** + * Returns user-specified metadata associated with this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + ImmutableMap userMetadata(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java new file mode 100644 index 000000000000..b4318be63f31 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -0,0 +1,189 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.collect.ImmutableSet; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Objects; +import java.util.Set; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Google Cloud Storage {@link FileSystem} + * + * @see + * Concepts and Terminology + * @see + * Bucket and Object Naming Guidelines + */ +@Immutable +public final class CloudStorageFileSystem extends FileSystem { + + /** + * Returns Google Cloud Storage {@link FileSystem} object for a given bucket name. + * + *

NOTE: You may prefer to use Java's standard API instead:

   {@code
+   *
+   *   FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));}
+ * + *

However some systems and build environments might be flaky when it comes to Java SPI. This + * is because services are generally runtime dependencies and depend on a META-INF file being + * present in your jar (generated by Google Auto at compile-time). In such cases, this method + * provides a simpler alternative. + * + * @see #forBucket(String, CloudStorageConfiguration) + * @see java.nio.file.FileSystems#getFileSystem(java.net.URI) + */ + public static CloudStorageFileSystem forBucket(String bucket) { + return forBucket(bucket, CloudStorageConfiguration.DEFAULT); + } + + /** + * Creates a new filesystem for a particular bucket, with customizable settings. + * + * @see #forBucket(String) + */ + public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfiguration config) { + checkArgument(!bucket.startsWith(URI_SCHEME + ":"), + "Bucket name must not have schema: %s", bucket); + return new CloudStorageFileSystem( + new CloudStorageFileSystemProvider(), bucket, checkNotNull(config)); + } + + public static final String URI_SCHEME = "gs"; + public static final String GCS_VIEW = "gcs"; + public static final String BASIC_VIEW = "basic"; + public static final int BLOCK_SIZE_DEFAULT = 2 * 1024 * 1024; + public static final FileTime FILE_TIME_UNKNOWN = FileTime.fromMillis(0); + public static final ImmutableSet SUPPORTED_VIEWS = ImmutableSet.of(BASIC_VIEW, GCS_VIEW); + + private final CloudStorageFileSystemProvider provider; + private final String bucket; + private final CloudStorageConfiguration config; + + CloudStorageFileSystem( + CloudStorageFileSystemProvider provider, + String bucket, + CloudStorageConfiguration config) { + checkArgument(!bucket.isEmpty(), "bucket"); + this.provider = provider; + this.bucket = bucket; + this.config = config; + } + + @Override + public CloudStorageFileSystemProvider provider() { + return provider; + } + + /** Returns the Cloud Storage bucket name being served by this file system. */ + public String bucket() { + return bucket; + } + + /** Returns the configuration object for this filesystem instance. */ + public CloudStorageConfiguration config() { + return config; + } + + /** Converts a cloud storage object name to a {@link Path} object. */ + @Override + public CloudStoragePath getPath(String first, String... more) { + checkArgument(!first.startsWith(URI_SCHEME + ":"), + "GCS FileSystem.getPath() must not have schema and bucket name: %s", first); + return CloudStoragePath.getPath(this, first, more); + } + + /** Does nothing. */ + @Override + public void close() {} + + /** Returns {@code true} */ + @Override + public boolean isOpen() { + return true; + } + + /** Returns {@code false} */ + @Override + public boolean isReadOnly() { + return false; + } + + /** Returns {@value UnixPath#SEPARATOR} */ + @Override + public String getSeparator() { + return "" + UnixPath.SEPARATOR; + } + + @Override + public Iterable getRootDirectories() { + return ImmutableSet.of(CloudStoragePath.getPath(this, UnixPath.ROOT)); + } + + @Override + public Iterable getFileStores() { + return ImmutableSet.of(); + } + + @Override + public Set supportedFileAttributeViews() { + return SUPPORTED_VIEWS; + } + + /** @throws UnsupportedOperationException */ + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + // TODO(b/18997520): Implement me. + throw new UnsupportedOperationException(); + } + + /** @throws UnsupportedOperationException */ + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + // TODO(b/18997520): Implement me. + throw new UnsupportedOperationException(); + } + + /** @throws UnsupportedOperationException */ + @Override + public WatchService newWatchService() throws IOException { + // TODO(b/18997520): Implement me. + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof CloudStorageFileSystem + && Objects.equals(config, ((CloudStorageFileSystem) other).config) + && Objects.equals(bucket, ((CloudStorageFileSystem) other).bucket); + } + + @Override + public int hashCode() { + return Objects.hash(bucket); + } + + @Override + public String toString() { + try { + return new URI(URI_SCHEME, bucket, null, null).toString(); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java new file mode 100644 index 000000000000..f1964b231ed1 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -0,0 +1,471 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Suppliers.memoize; +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.URI_SCHEME; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.buildFileOptions; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkBucket; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkNotNullArray; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkPath; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.copyFileOptions; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.stripPathFromUri; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.appengine.tools.cloudstorage.GcsFileOptions; +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.appengine.tools.cloudstorage.GcsInputChannel; +import com.google.appengine.tools.cloudstorage.GcsOutputChannel; +import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.auto.service.AutoService; +import com.google.common.base.MoreObjects; +import com.google.common.base.Supplier; +import com.google.common.primitives.Ints; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileStore; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** Google Cloud Storage {@link FileSystemProvider} */ +@ThreadSafe +@AutoService(FileSystemProvider.class) +public final class CloudStorageFileSystemProvider extends FileSystemProvider { + + private static final Supplier gcsServiceSupplier = + memoize(new Supplier() { + @Override + public GcsService get() { + return GcsServiceFactory.createGcsService(); + }}); + + private final GcsService gcsService; + + /** + * Default constructor which should only be called by Java SPI. + * + * @see java.nio.file.FileSystems#getFileSystem(URI) + * @see CloudStorageFileSystem#forBucket(String) + */ + public CloudStorageFileSystemProvider() { + this(gcsServiceSupplier.get()); + } + + private CloudStorageFileSystemProvider(GcsService gcsService) { + this.gcsService = checkNotNull(gcsService); + } + + GcsService getGcsService() { + return gcsService; + } + + @Override + public String getScheme() { + return URI_SCHEME; + } + + /** Returns cloud storage file system, provided a URI with no path, e.g. {@code gs://bucket} */ + @Override + public CloudStorageFileSystem getFileSystem(URI uri) { + return newFileSystem(uri, Collections.emptyMap()); + } + + /** Returns cloud storage file system, provided a URI with no path, e.g. {@code gs://bucket} */ + @Override + public CloudStorageFileSystem newFileSystem(URI uri, Map env) { + checkArgument(uri.getScheme().equalsIgnoreCase(URI_SCHEME), + "Cloud Storage URIs must have '%s' scheme: %s", URI_SCHEME, uri); + checkArgument(!isNullOrEmpty(uri.getHost()), + "%s:// URIs must have a host: %s", URI_SCHEME, uri); + checkArgument(uri.getPort() == -1 + && isNullOrEmpty(uri.getPath()) + && isNullOrEmpty(uri.getQuery()) + && isNullOrEmpty(uri.getFragment()) + && isNullOrEmpty(uri.getUserInfo()), + "GCS FileSystem URIs mustn't have: port, userinfo, path, query, or fragment: %s", uri); + checkBucket(uri.getHost()); + return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env)); + } + + @Override + public CloudStoragePath getPath(URI uri) { + return CloudStoragePath.getPath(getFileSystem(stripPathFromUri(uri)), uri.getPath()); + } + + @Override + public SeekableByteChannel newByteChannel( + Path path, Set options, FileAttribute... attrs) throws IOException { + checkNotNull(path); + checkNotNullArray(attrs); + if (options.contains(StandardOpenOption.WRITE)) { + // TODO(b/18997618): Make our OpenOptions implement FileAttribute. Also remove buffer option. + return newWriteChannel(path, options); + } else { + return newReadChannel(path, options); + } + } + + private SeekableByteChannel newReadChannel( + Path path, Set options) throws IOException { + for (OpenOption option : options) { + if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case READ: + // Default behavior. + break; + case SPARSE: + case TRUNCATE_EXISTING: + // Ignored by specification. + break; + case WRITE: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case CREATE: + case CREATE_NEW: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + CloudStoragePath cloudPath = checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + return CloudStorageReadChannel.create(gcsService, cloudPath.getGcsFilename(), 0); + } + + private SeekableByteChannel newWriteChannel( + Path path, Set options) throws IOException { + boolean wantCreateNew = false; + for (OpenOption option : options) { + if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case CREATE: + case TRUNCATE_EXISTING: + case WRITE: + // Default behavior. + break; + case SPARSE: + // Ignored by specification. + break; + case CREATE_NEW: + wantCreateNew = true; + break; + case READ: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + // These will be interpreted later. + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + CloudStoragePath cloudPath = checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + if (wantCreateNew) { + // XXX: Java's documentation says this should be atomic. + if (gcsService.getMetadata(cloudPath.getGcsFilename()) != null) { + throw new FileAlreadyExistsException(cloudPath.toString()); + } + } + GcsFilename file = cloudPath.getGcsFilename(); + GcsFileOptions fileOptions = buildFileOptions(new GcsFileOptions.Builder(), options.toArray()); + return new CloudStorageWriteChannel(gcsService.createOrReplace(file, fileOptions)); + } + + @Override + public InputStream newInputStream(Path path, OpenOption... options) throws IOException { + InputStream result = super.newInputStream(path, options); + CloudStoragePath cloudPath = checkPath(path); + int blockSize = cloudPath.getFileSystem().config().blockSize(); + for (OpenOption option : options) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } + } + return new BufferedInputStream(result, blockSize); + } + + @Override + public final boolean deleteIfExists(Path path) throws IOException { + CloudStoragePath cloudPath = checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + return gcsService.delete(cloudPath.getGcsFilename()); + } + + @Override + public void delete(Path path) throws IOException { + CloudStoragePath cloudPath = checkPath(path); + if (!deleteIfExists(cloudPath)) { + throw new NoSuchFileException(cloudPath.toString()); + } + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + for (CopyOption option : options) { + if (option == StandardCopyOption.ATOMIC_MOVE) { + throw new AtomicMoveNotSupportedException(source.toString(), target.toString(), + "Google Cloud Storage does not support atomic move operations."); + } + } + copy(source, target, options); + delete(source); + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + boolean wantCopyAttributes = false; + boolean wantReplaceExisting = false; + int blockSize = -1; + for (CopyOption option : options) { + if (option instanceof StandardCopyOption) { + switch ((StandardCopyOption) option) { + case COPY_ATTRIBUTES: + wantCopyAttributes = true; + break; + case REPLACE_EXISTING: + wantReplaceExisting = true; + break; + case ATOMIC_MOVE: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } + // The rest will be interpreted later. + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + CloudStoragePath fromPath = checkPath(source); + CloudStoragePath toPath = checkPath(target); + + blockSize = blockSize != -1 ? blockSize + : Ints.max(fromPath.getFileSystem().config().blockSize(), + toPath.getFileSystem().config().blockSize()); + if (fromPath.seemsLikeADirectory() && toPath.seemsLikeADirectory()) { + if (fromPath.getFileSystem().config().usePseudoDirectories() + && toPath.getFileSystem().config().usePseudoDirectories()) { + // NOOP: This would normally create an empty directory. + return; + } else { + checkArgument(!fromPath.getFileSystem().config().usePseudoDirectories() + && !toPath.getFileSystem().config().usePseudoDirectories(), + "File systems associated with paths don't agree on pseudo-directories."); + } + } + if (fromPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(fromPath); + } + if (toPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(toPath); + } + GcsFilename from = fromPath.getGcsFilename(); + GcsFilename to = toPath.getGcsFilename(); + GcsFileMetadata metadata = gcsService.getMetadata(from); + if (metadata == null) { + throw new NoSuchFileException(source.toString()); + } + if (fromPath.equals(toPath)) { + return; + } + if (!wantReplaceExisting && gcsService.getMetadata(to) != null) { + throw new FileAlreadyExistsException(target.toString()); + } + GcsFileOptions.Builder builder = wantCopyAttributes + ? copyFileOptions(metadata.getOptions()) + : new GcsFileOptions.Builder(); + GcsFileOptions fileOptions = buildFileOptions(builder, options); + try (GcsInputChannel input = gcsService.openReadChannel(from, 0); + GcsOutputChannel output = gcsService.createOrReplace(to, fileOptions)) { + ByteBuffer block = ByteBuffer.allocate(blockSize); + while (input.read(block) != -1) { + block.flip(); + while (block.hasRemaining()) { + output.write(block); + } + block.clear(); + } + } + } + + @Override + public boolean isSameFile(Path path, Path path2) { + return checkPath(path).equals(checkPath(path2)); + } + + /** Returns {@code false} */ + @Override + public boolean isHidden(Path path) { + checkPath(path); + return false; + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + for (AccessMode mode : modes) { + switch (mode) { + case READ: + case WRITE: + break; + case EXECUTE: + default: + throw new UnsupportedOperationException(mode.toString()); + } + } + CloudStoragePath cloudPath = checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + return; + } + if (gcsService.getMetadata(cloudPath.getGcsFilename()) == null) { + throw new NoSuchFileException(path.toString()); + } + } + + @Override + public A readAttributes( + Path path, Class type, LinkOption... options) throws IOException { + CloudStoragePath cloudPath = checkPath(path); + checkNotNull(type); + checkNotNullArray(options); + if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) { + throw new UnsupportedOperationException(type.getSimpleName()); + } + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + @SuppressWarnings("unchecked") + A result = (A) CloudStoragePseudoDirectoryAttributes.SINGLETON_INSTANCE; + return result; + } + GcsFileMetadata metadata = gcsService.getMetadata(cloudPath.getGcsFilename()); + if (metadata == null) { + throw new NoSuchFileException(path.toString()); + } + @SuppressWarnings("unchecked") + A result = (A) new CloudStorageObjectAttributes(metadata); + return result; + } + + @Override + public V getFileAttributeView( + Path path, Class type, LinkOption... options) { + CloudStoragePath cloudPath = checkPath(path); + checkNotNull(type); + checkNotNullArray(options); + if (type != CloudStorageFileAttributeView.class && type != BasicFileAttributeView.class) { + throw new UnsupportedOperationException(type.getSimpleName()); + } + @SuppressWarnings("unchecked") + V result = (V) new CloudStorageFileAttributeView(this, cloudPath); + return result; + } + + /** Does nothing since GCS uses fake directories. */ + @Override + public void createDirectory(Path dir, FileAttribute... attrs) { + checkPath(dir); + checkNotNullArray(attrs); + } + + /** @throws UnsupportedOperationException */ + @Override + public DirectoryStream newDirectoryStream(Path dir, Filter filter) { + // TODO(b/18997618): Implement me. + throw new UnsupportedOperationException(); + } + + /** + * This feature is not supported. Please use {@link #readAttributes(Path, Class, LinkOption...)} + * + * @throws UnsupportedOperationException + */ + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) { + throw new UnsupportedOperationException(); + } + + /** + * This feature is not supported, since Cloud Storage objects are immutable. + * + * @throws UnsupportedOperationException + */ + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { + throw new CloudStorageObjectImmutableException(); + } + + /** + * This feature is not supported. + * + * @throws UnsupportedOperationException + */ + @Override + public FileStore getFileStore(Path path) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof CloudStorageFileSystemProvider + && Objects.equals(gcsService, ((CloudStorageFileSystemProvider) other).gcsService); + } + + @Override + public int hashCode() { + return Objects.hash(gcsService); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("gcsService", gcsService) + .toString(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java new file mode 100644 index 000000000000..b3265cd6276b --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectAttributes.java @@ -0,0 +1,155 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.FileTime; +import java.util.Objects; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** Metadata for a Google Cloud Storage object. */ +@Immutable +final class CloudStorageObjectAttributes implements CloudStorageFileAttributes { + + private final GcsFileMetadata metadata; + + CloudStorageObjectAttributes(GcsFileMetadata metadata) { + this.metadata = checkNotNull(metadata); + } + + @Override + public long size() { + return metadata.getLength(); + } + + @Override + public FileTime creationTime() { + if (metadata.getLastModified() == null) { + return FILE_TIME_UNKNOWN; + } + return FileTime.fromMillis(metadata.getLastModified().getTime()); + } + + @Override + public FileTime lastModifiedTime() { + return creationTime(); + } + + /** Returns the HTTP etag hash for this object. */ + @Override + public Optional etag() { + return Optional.fromNullable(metadata.getEtag()); + } + + /** Returns the mime type (e.g. text/plain) if it was set for this object. */ + @Override + public Optional mimeType() { + return Optional.fromNullable(metadata.getOptions().getMimeType()); + } + + /** + * Returns the ACL value on this Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + @Override + public Optional acl() { + return Optional.fromNullable(metadata.getOptions().getAcl()); + } + + /** + * Returns the {@code Cache-Control} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + @Override + public Optional cacheControl() { + return Optional.fromNullable(metadata.getOptions().getCacheControl()); + } + + /** + * Returns the {@code Content-Encoding} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + @Override + public Optional contentEncoding() { + return Optional.fromNullable(metadata.getOptions().getContentEncoding()); + } + + /** + * Returns the {@code Content-Disposition} HTTP header value, if set on this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + @Override + public Optional contentDisposition() { + return Optional.fromNullable(metadata.getOptions().getContentDisposition()); + } + + /** + * Returns user-specified metadata associated with this object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + @Override + public ImmutableMap userMetadata() { + return ImmutableMap.copyOf(metadata.getOptions().getUserMetadata()); + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public FileTime lastAccessTime() { + return FILE_TIME_UNKNOWN; + } + + @Override + public Object fileKey() { + return metadata.getFilename(); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof CloudStorageObjectAttributes + && Objects.equals(metadata, ((CloudStorageObjectAttributes) other).metadata); + } + + @Override + public int hashCode() { + return Objects.hash(metadata); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("metadata", metadata) + .toString(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectImmutableException.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectImmutableException.java new file mode 100644 index 000000000000..669f1c9ae91b --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageObjectImmutableException.java @@ -0,0 +1,8 @@ +package com.google.gcloud.storage.contrib.nio; + +/** Exception thrown to indicate we don't support a mutation of a cloud storage object. */ +public final class CloudStorageObjectImmutableException extends UnsupportedOperationException { + CloudStorageObjectImmutableException() { + super("Cloud Storage objects are immutable."); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java new file mode 100644 index 000000000000..4142cef2bb6a --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOption.java @@ -0,0 +1,17 @@ +package com.google.gcloud.storage.contrib.nio; + +import java.nio.file.CopyOption; +import java.nio.file.OpenOption; + +/** Master interface for file operation option classes related to Google Cloud Storage. */ +public interface CloudStorageOption { + + /** Interface for GCS options that can be specified when opening files. */ + public interface Open extends CloudStorageOption, OpenOption {} + + /** Interface for GCS options that can be specified when copying files. */ + public interface Copy extends CloudStorageOption, CopyOption {} + + /** Interface for GCS options that can be specified when opening or copying files. */ + public interface OpenCopy extends Open, Copy {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java new file mode 100644 index 000000000000..c66842d44c03 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptions.java @@ -0,0 +1,71 @@ +package com.google.gcloud.storage.contrib.nio; + +/** Helper class for specifying options when opening and copying Cloud Storage files. */ +public final class CloudStorageOptions { + + /** Sets the mime type header on an object, e.g. {@code "text/plain"}. */ + public static CloudStorageOption.OpenCopy withMimeType(String mimeType) { + return OptionMimeType.create(mimeType); + } + + /** Disables caching on an object. Same as: {@code withCacheControl("no-cache")}. */ + public static CloudStorageOption.OpenCopy withoutCaching() { + return withCacheControl("no-cache"); + } + + /** + * Sets the {@code Cache-Control} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + public static CloudStorageOption.OpenCopy withCacheControl(String cacheControl) { + return OptionCacheControl.create(cacheControl); + } + + /** + * Sets the {@code Content-Disposition} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + public static CloudStorageOption.OpenCopy withContentDisposition(String contentDisposition) { + return OptionContentDisposition.create(contentDisposition); + } + + /** + * Sets the {@code Content-Encoding} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + public static CloudStorageOption.OpenCopy withContentEncoding(String contentEncoding) { + return OptionContentEncoding.create(contentEncoding); + } + + /** + * Sets the ACL value on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + public static CloudStorageOption.OpenCopy withAcl(String acl) { + return OptionAcl.create(acl); + } + + /** + * Sets an unmodifiable piece of user metadata on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#xgoogmeta" + */ + public static CloudStorageOption.OpenCopy withUserMetadata(String key, String value) { + return OptionUserMetadata.create(key, value); + } + + /** + * Sets the block size (in bytes) when talking to the GCS server. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public static CloudStorageOption.OpenCopy withBlockSize(int size) { + return OptionBlockSize.create(size); + } + + private CloudStorageOptions() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java new file mode 100644 index 000000000000..7d8e88753ce8 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePath.java @@ -0,0 +1,332 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.URI_SCHEME; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkNotNullArray; +import static com.google.gcloud.storage.contrib.nio.CloudStorageUtil.checkPath; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.collect.UnmodifiableIterator; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchEvent.Modifier; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Collections; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Google Cloud Storage {@link Path} + * + * @see UnixPath + */ +@Immutable +public final class CloudStoragePath implements Path { + + private static final Pattern EXTRA_SLASHES_OR_DOT_DIRS_PATTERN = + Pattern.compile("^\\.\\.?/|//|/\\.\\.?/|/\\.\\.?$"); + + private final CloudStorageFileSystem fileSystem; + private final UnixPath path; + + private CloudStoragePath(CloudStorageFileSystem fileSystem, UnixPath path) { + this.fileSystem = fileSystem; + this.path = path; + } + + static CloudStoragePath getPath( + CloudStorageFileSystem fileSystem, String path, String... more) { + return new CloudStoragePath( + fileSystem, UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), path, more)); + } + + /** Returns the Cloud Storage bucket name being served by this file system. */ + public String bucket() { + return fileSystem.bucket(); + } + + /** Returns path converted to a {@link GcsFilename} so I/O can be performed. */ + GcsFilename getGcsFilename() { + return new GcsFilename(bucket(), toRealPath().path.toString()); + } + + boolean seemsLikeADirectory() { + return path.seemsLikeADirectory(); + } + + boolean seemsLikeADirectoryAndUsePseudoDirectories() { + return path.seemsLikeADirectory() && fileSystem.config().usePseudoDirectories(); + } + + @Override + public CloudStorageFileSystem getFileSystem() { + return fileSystem; + } + + @Nullable + @Override + public CloudStoragePath getRoot() { + return newPath(path.getRoot()); + } + + @Override + public boolean isAbsolute() { + return path.isAbsolute(); + } + + /** + * Changes relative path to absolute, using + * {@link CloudStorageConfiguration#workingDirectory() workingDirectory} as the current dir. + */ + @Override + public CloudStoragePath toAbsolutePath() { + return newPath(path.toAbsolutePath(getWorkingDirectory())); + } + + /** + * Returns this path rewritten to the Cloud Storage object name that'd be used to perform i/o. + * + *

This method makes path {@link #toAbsolutePath() absolute} and removes the prefix slash from + * the absolute path when {@link CloudStorageConfiguration#stripPrefixSlash() stripPrefixSlash} + * is {@code true}. + * + * @throws IllegalArgumentException if path contains extra slashes or dot-dirs when + * {@link CloudStorageConfiguration#permitEmptyPathComponents() permitEmptyPathComponents} + * is {@code false}, or if the resulting path is empty. + */ + @Override + public CloudStoragePath toRealPath(LinkOption... options) { + checkNotNullArray(options); + return newPath(toRealPathInternal(true)); + } + + private UnixPath toRealPathInternal(boolean errorCheck) { + UnixPath objectName = path.toAbsolutePath(getWorkingDirectory()); + if (errorCheck && !fileSystem.config().permitEmptyPathComponents()) { + checkArgument(!EXTRA_SLASHES_OR_DOT_DIRS_PATTERN.matcher(objectName).find(), + "I/O not allowed on dot-dirs or extra slashes when !permitEmptyPathComponents: %s", + objectName); + } + if (fileSystem.config().stripPrefixSlash()) { + objectName = objectName.removeBeginningSeparator(); + } + checkArgument(!errorCheck || !objectName.isEmpty(), + "I/O not allowed on empty GCS object names."); + return objectName; + } + + /** + * Returns path without extra slashes or {@code .} and {@code ..} and preserves trailing slash. + * + * @see java.nio.file.Path#normalize() + */ + @Override + public CloudStoragePath normalize() { + return newPath(path.normalize()); + } + + @Override + public CloudStoragePath resolve(Path object) { + return newPath(path.resolve(checkPath(object).path)); + } + + @Override + public CloudStoragePath resolve(String other) { + return newPath(path.resolve(getUnixPath(other))); + } + + @Override + public CloudStoragePath resolveSibling(Path other) { + return newPath(path.resolveSibling(checkPath(other).path)); + } + + @Override + public CloudStoragePath resolveSibling(String other) { + return newPath(path.resolveSibling(getUnixPath(other))); + } + + @Override + public CloudStoragePath relativize(Path object) { + return newPath(path.relativize(checkPath(object).path)); + } + + @Nullable + @Override + public CloudStoragePath getParent() { + return newPath(path.getParent()); + } + + @Nullable + @Override + public CloudStoragePath getFileName() { + return newPath(path.getFileName()); + } + + @Override + public CloudStoragePath subpath(int beginIndex, int endIndex) { + return newPath(path.subpath(beginIndex, endIndex)); + } + + @Override + public int getNameCount() { + return path.getNameCount(); + } + + @Override + public CloudStoragePath getName(int index) { + return newPath(path.getName(index)); + } + + @Override + public boolean startsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.startsWith(that.path); + } + + @Override + public boolean startsWith(String other) { + return path.startsWith(getUnixPath(other)); + } + + @Override + public boolean endsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.endsWith(that.path); + } + + @Override + public boolean endsWith(String other) { + return path.endsWith(getUnixPath(other)); + } + + /** @throws UnsupportedOperationException */ + @Override + public WatchKey register(WatchService watcher, Kind[] events, Modifier... modifiers) { + // TODO(b/18998105): Implement me. + throw new UnsupportedOperationException(); + } + + /** @throws UnsupportedOperationException */ + @Override + public WatchKey register(WatchService watcher, Kind... events) { + // TODO(b/18998105): Implement me. + throw new UnsupportedOperationException(); + } + + /** + * This operation is not supported, since GCS files aren't backed by the local file system. + * + * @throws UnsupportedOperationException + */ + @Override + public File toFile() { + throw new UnsupportedOperationException("GCS objects aren't available locally"); + } + + @Override + public Iterator iterator() { + if (path.isEmpty()) { + return Collections.singleton(this).iterator(); + } else if (path.isRoot()) { + return Collections.emptyIterator(); + } else { + return new PathIterator(); + } + } + + @Override + public int compareTo(Path other) { + // Documented to throw CCE if other is associated with a different FileSystemProvider. + CloudStoragePath that = (CloudStoragePath) other; + int res = bucket().compareTo(that.bucket()); + if (res != 0) { + return res; + } + return toRealPathInternal(false).compareTo(that.toRealPathInternal(false)); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof CloudStoragePath + && Objects.equals(bucket(), ((CloudStoragePath) other).bucket()) + && Objects.equals(toRealPathInternal(false), + ((CloudStoragePath) other).toRealPathInternal(false)); + } + + @Override + public int hashCode() { + return Objects.hash(bucket(), toRealPathInternal(false)); + } + + @Override + public String toString() { + return path.toString(); + } + + @Override + public URI toUri() { + try { + return new URI(URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } + + @Nullable + private CloudStoragePath newPath(@Nullable UnixPath newPath) { + if (newPath == path) { // Nonuse of equals is intentional. + return this; + } else if (newPath != null) { + return new CloudStoragePath(fileSystem, newPath); + } else { + return null; + } + } + + private UnixPath getUnixPath(String newPath) { + return UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), newPath); + } + + private UnixPath getWorkingDirectory() { + return getUnixPath(fileSystem.config().workingDirectory()); + } + + /** Transform iterator providing a slight performance boost over {@code FluentIterable}. */ + private final class PathIterator extends UnmodifiableIterator { + private final Iterator delegate = path.split(); + + @Override + public Path next() { + return newPath(getUnixPath(delegate.next())); + } + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java new file mode 100644 index 000000000000..fc381de3244b --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java @@ -0,0 +1,97 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.FileTime; + +/** Metadata for a cloud storage pseudo-directory. */ +final class CloudStoragePseudoDirectoryAttributes implements CloudStorageFileAttributes { + + static final CloudStoragePseudoDirectoryAttributes SINGLETON_INSTANCE = + new CloudStoragePseudoDirectoryAttributes(); + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public Object fileKey() { + return null; + } + + @Override + public long size() { + return 1; // Allow I/O to happen before we fail. + } + + @Override + public FileTime lastModifiedTime() { + return FILE_TIME_UNKNOWN; + } + + @Override + public FileTime creationTime() { + return FILE_TIME_UNKNOWN; + } + + @Override + public FileTime lastAccessTime() { + return FILE_TIME_UNKNOWN; + } + + @Override + public Optional etag() { + return Optional.absent(); + } + + @Override + public Optional mimeType() { + return Optional.absent(); + } + + @Override + public Optional acl() { + return Optional.absent(); + } + + @Override + public Optional cacheControl() { + return Optional.absent(); + } + + @Override + public Optional contentEncoding() { + return Optional.absent(); + } + + @Override + public Optional contentDisposition() { + return Optional.absent(); + } + + @Override + public ImmutableMap userMetadata() { + return ImmutableMap.of(); + } + + private CloudStoragePseudoDirectoryAttributes() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java new file mode 100644 index 000000000000..d4ebbdad6e20 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java @@ -0,0 +1,10 @@ +package com.google.gcloud.storage.contrib.nio; + +import java.nio.file.InvalidPathException; + +/** Exception thrown when erroneously trying to operate on a path with a trailing slash. */ +public final class CloudStoragePseudoDirectoryException extends InvalidPathException { + CloudStoragePseudoDirectoryException(CloudStoragePath path) { + super(path.toString(), "Can't perform I/O on pseudo-directories (trailing slash)"); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java new file mode 100644 index 000000000000..f4a29dde941c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannel.java @@ -0,0 +1,137 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.appengine.tools.cloudstorage.GcsInputChannel; +import com.google.appengine.tools.cloudstorage.GcsService; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.NoSuchFileException; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage read channel. + * + * @see CloudStorageWriteChannel + */ +@ThreadSafe +final class CloudStorageReadChannel implements SeekableByteChannel { + + static CloudStorageReadChannel create( + GcsService gcsService, GcsFilename file, long position) throws IOException { + // XXX: Reading size and opening file should be atomic. + long size = fetchSize(gcsService, file); + return new CloudStorageReadChannel(gcsService, file, position, size, + gcsService.openReadChannel(file, position)); + } + + private final GcsService gcsService; + private final GcsFilename file; + private long position; + private long size; + private GcsInputChannel channel; + + private CloudStorageReadChannel( + GcsService gcsService, GcsFilename file, long position, long size, GcsInputChannel channel) { + this.gcsService = gcsService; + this.file = file; + this.position = position; + this.size = size; + this.channel = channel; + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + synchronized (this) { + checkOpen(); + int amt = channel.read(dst); + if (amt > 0) { + position += amt; + // XXX: This would only ever happen if the fetchSize() race-condition occurred. + if (position > size) { + size = position; + } + } + return amt; + } + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + checkArgument(newPosition >= 0); + synchronized (this) { + checkOpen(); + if (newPosition == position) { + return this; + } + position = newPosition; + size = fetchSize(gcsService, file); + channel.close(); + channel = gcsService.openReadChannel(file, position); + return this; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new NonWritableChannelException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } + + private static long fetchSize(GcsService gcsService, GcsFilename file) throws IOException, + NoSuchFileException { + GcsFileMetadata metadata = gcsService.getMetadata(file); + if (metadata == null) { + throw new NoSuchFileException( + String.format("gs://%s/%s", file.getBucketName(), file.getObjectName())); + } + return metadata.getLength(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java new file mode 100644 index 000000000000..79f46af172ee --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageUtil.java @@ -0,0 +1,110 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.google.appengine.tools.cloudstorage.GcsFileOptions; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.util.Map; +import java.util.regex.Pattern; + +final class CloudStorageUtil { + + private static final Pattern BUCKET_PATTERN = Pattern.compile("[a-z0-9][-._a-z0-9.]+[a-z0-9]"); + + static void checkBucket(String bucket) { + // TODO(b/18998200): The true check is actually more complicated. Consider implementing it. + checkArgument(BUCKET_PATTERN.matcher(bucket).matches(), "" + + "Invalid bucket name: '" + bucket + "'. " + + "GCS bucket names must contain only lowercase letters, numbers, dashes (-), " + + "underscores (_), and dots (.). Bucket names must start and end with a number or letter. " + + "See https://developers.google.com/storage/docs/bucketnaming for more details."); + } + + static CloudStoragePath checkPath(Path path) { + if (!(checkNotNull(path) instanceof CloudStoragePath)) { + throw new ProviderMismatchException(String.format( + "Not a cloud storage path: %s (%s)", path, path.getClass().getSimpleName())); + } + return (CloudStoragePath) path; + } + + static GcsFileOptions.Builder copyFileOptions(GcsFileOptions options) { + GcsFileOptions.Builder builder = new GcsFileOptions.Builder(); + if (!isNullOrEmpty(options.getAcl())) { + builder.acl(options.getAcl()); + } + if (!isNullOrEmpty(options.getCacheControl())) { + builder.cacheControl(options.getCacheControl()); + } + if (!isNullOrEmpty(options.getContentDisposition())) { + builder.contentDisposition(options.getContentDisposition()); + } + if (!isNullOrEmpty(options.getContentEncoding())) { + builder.contentEncoding(options.getContentEncoding()); + } + if (!isNullOrEmpty(options.getMimeType())) { + builder.mimeType(options.getMimeType()); + } + for (Map.Entry entry : options.getUserMetadata().entrySet()) { + builder.addUserMetadata(entry.getKey(), entry.getValue()); + } + return builder; + } + + @SafeVarargs + static GcsFileOptions buildFileOptions(GcsFileOptions.Builder builder, T... options) { + for (Object option : options) { + if (option instanceof OptionAcl) { + builder.acl(((OptionAcl) option).acl()); + } else if (option instanceof OptionCacheControl) { + builder.cacheControl(((OptionCacheControl) option).cacheControl()); + } else if (option instanceof OptionContentDisposition) { + builder.contentDisposition(((OptionContentDisposition) option).contentDisposition()); + } else if (option instanceof OptionContentEncoding) { + builder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); + } else if (option instanceof OptionMimeType) { + builder.mimeType(((OptionMimeType) option).mimeType()); + } else if (option instanceof OptionUserMetadata) { + OptionUserMetadata metadata = (OptionUserMetadata) option; + builder.addUserMetadata(metadata.key(), metadata.value()); + } + } + return builder.build(); + } + + static URI stripPathFromUri(URI uri) { + try { + return new URI( + uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + null, + uri.getQuery(), + uri.getFragment()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + + /** Makes NullPointerTester happy. */ + @SafeVarargs + static void checkNotNullArray(T... values) { + for (T value : values) { + checkNotNull(value); + } + } + + static boolean getPropertyBoolean(String property, boolean defaultValue) { + String value = System.getProperty(property); + return value != null ? Boolean.valueOf(value) : defaultValue; + } + + private CloudStorageUtil() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java new file mode 100644 index 000000000000..2b5a004e1bc4 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannel.java @@ -0,0 +1,96 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.appengine.tools.cloudstorage.GcsOutputChannel; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; +import java.nio.channels.SeekableByteChannel; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage write channel. + * + *

This class does not support seeking, reading, or append. + * + * @see CloudStorageReadChannel + */ +@ThreadSafe +final class CloudStorageWriteChannel implements SeekableByteChannel { + + private GcsOutputChannel channel; + private long position; + private long size; + + CloudStorageWriteChannel(GcsOutputChannel channel) { + this.channel = channel; + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + throw new NonReadableChannelException(); + } + + @Override + public int write(ByteBuffer src) throws IOException { + synchronized (this) { + checkOpen(); + int amt = channel.write(src); + if (amt > 0) { + position += amt; + size += amt; + } + return amt; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public SeekableByteChannel truncate(long newSize) throws IOException { + // TODO(b/18997913): Emulate this functionality by closing and rewriting old file up to newSize. + // Or maybe just swap out GcsService for the Apiary client. + throw new UnsupportedOperationException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java new file mode 100644 index 000000000000..c0068ee1c61d --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionAcl.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionAcl implements CloudStorageOption.OpenCopy { + + static OptionAcl create(String acl) { + return new AutoValue_OptionAcl(acl); + } + + abstract String acl(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionBlockSize.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionBlockSize.java new file mode 100644 index 000000000000..f33f601fad61 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionBlockSize.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionBlockSize implements CloudStorageOption.OpenCopy { + + static OptionBlockSize create(int size) { + return new AutoValue_OptionBlockSize(size); + } + + abstract int size(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionCacheControl.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionCacheControl.java new file mode 100644 index 000000000000..c33d4394fa24 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionCacheControl.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionCacheControl implements CloudStorageOption.OpenCopy { + + static OptionCacheControl create(String cacheControl) { + return new AutoValue_OptionCacheControl(cacheControl); + } + + abstract String cacheControl(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentDisposition.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentDisposition.java new file mode 100644 index 000000000000..7bebefcba19c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentDisposition.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentDisposition implements CloudStorageOption.OpenCopy { + + static OptionContentDisposition create(String contentDisposition) { + return new AutoValue_OptionContentDisposition(contentDisposition); + } + + abstract String contentDisposition(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentEncoding.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentEncoding.java new file mode 100644 index 000000000000..879d05983f37 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionContentEncoding.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentEncoding implements CloudStorageOption.OpenCopy { + + static OptionContentEncoding create(String contentEncoding) { + return new AutoValue_OptionContentEncoding(contentEncoding); + } + + abstract String contentEncoding(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionMimeType.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionMimeType.java new file mode 100644 index 000000000000..feb1b0c6dd1b --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionMimeType.java @@ -0,0 +1,13 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionMimeType implements CloudStorageOption.OpenCopy { + + static OptionMimeType create(String mimeType) { + return new AutoValue_OptionMimeType(mimeType); + } + + abstract String mimeType(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java new file mode 100644 index 000000000000..9851d2ad5c4f --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/OptionUserMetadata.java @@ -0,0 +1,14 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionUserMetadata implements CloudStorageOption.OpenCopy { + + static OptionUserMetadata create(String key, String value) { + return new AutoValue_OptionUserMetadata(key, value); + } + + abstract String key(); + abstract String value(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java new file mode 100644 index 000000000000..7b80a6299be6 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/UnixPath.java @@ -0,0 +1,494 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.common.collect.PeekingIterator; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Unix file system path. + * + *

This class is helpful for writing {@link java.nio.file.Path Path} implementations. + * + *

This implementation behaves almost identically to {@code sun.nio.fs.UnixPath}. The only + * difference is that some methods (like {@link #relativize(UnixPath)} go to greater lengths to + * preserve trailing backslashes, in order to ensure the path will continue to be recognized as a + * directory. + * + *

NOTE: This code might not play nice with + * Supplementary + * Characters as Surrogates. + */ +@Immutable +final class UnixPath implements CharSequence { + + public static final char DOT = '.'; + public static final char SEPARATOR = '/'; + public static final String ROOT = "" + SEPARATOR; + public static final String CURRENT_DIR = "" + DOT; + public static final String PARENT_DIR = "" + DOT + DOT; + public static final UnixPath EMPTY_PATH = new UnixPath(false, ""); + public static final UnixPath ROOT_PATH = new UnixPath(false, ROOT); + + private static final Splitter SPLITTER = Splitter.on(SEPARATOR).omitEmptyStrings(); + private static final Splitter SPLITTER_PERMIT_EMPTY_COMPONENTS = Splitter.on(SEPARATOR); + private static final Joiner JOINER = Joiner.on(SEPARATOR); + private static final Ordering> ORDERING = Ordering.natural().lexicographical(); + + private final String path; + private List lazyStringParts; + private final boolean permitEmptyComponents; + + private UnixPath(boolean permitEmptyComponents, String path) { + this.path = checkNotNull(path); + this.permitEmptyComponents = permitEmptyComponents; + } + + /** Returns new UnixPath of {@code first}. */ + public static UnixPath getPath(boolean permitEmptyComponents, String path) { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRootInternal(path)) { + return ROOT_PATH; + } else { + return new UnixPath(permitEmptyComponents, path); + } + } + + /** + * Returns new UnixPath of {@code first} with {@code more} components resolved against it. + * + * @see #resolve(UnixPath) + * @see java.nio.file.FileSystem#getPath(String, String...) + */ + public static UnixPath getPath(boolean permitEmptyComponents, String first, String... more) { + if (more.length == 0) { + return getPath(permitEmptyComponents, first); + } + StringBuilder builder = new StringBuilder(first); + for (int i = 0; i < more.length; i++) { + String part = more[i]; + if (part.isEmpty()) { + continue; + } else if (isAbsoluteInternal(part)) { + if (i == more.length - 1) { + return new UnixPath(permitEmptyComponents, part); + } else { + builder.replace(0, builder.length(), part); + } + } else if (hasTrailingSeparatorInternal(builder)) { + builder.append(part); + } else { + builder.append(SEPARATOR); + builder.append(part); + } + } + return new UnixPath(permitEmptyComponents, builder.toString()); + } + + /** Returns {@code true} consists only of {@code separator}. */ + public boolean isRoot() { + return isRootInternal(path); + } + + private static boolean isRootInternal(String path) { + return path.length() == 1 && path.charAt(0) == SEPARATOR; + } + + /** Returns {@code true} if path starts with {@code separator}. */ + public boolean isAbsolute() { + return isAbsoluteInternal(path); + } + + private static boolean isAbsoluteInternal(String path) { + return !path.isEmpty() && path.charAt(0) == SEPARATOR; + } + + /** Returns {@code true} if path ends with {@code separator}. */ + public boolean hasTrailingSeparator() { + return hasTrailingSeparatorInternal(path); + } + + private static boolean hasTrailingSeparatorInternal(CharSequence path) { + return path.length() != 0 && path.charAt(path.length() - 1) == SEPARATOR; + } + + /** Returns {@code true} if path ends with a trailing slash, or would after normalization. */ + public boolean seemsLikeADirectory() { + int length = path.length(); + return path.isEmpty() + || path.charAt(length - 1) == SEPARATOR + || path.endsWith(".") && (length == 1 || path.charAt(length - 2) == SEPARATOR) + || path.endsWith("..") && (length == 2 || path.charAt(length - 3) == SEPARATOR); + } + + /** + * Returns last component in {@code path}. + * + * @see java.nio.file.Path#getFileName() + */ + @Nullable + public UnixPath getFileName() { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRoot()) { + return null; + } else { + List parts = getParts(); + String last = parts.get(parts.size() - 1); + return parts.size() == 1 && path.equals(last) + ? this : new UnixPath(permitEmptyComponents, last); + } + } + + /** + * Returns parent directory (including trailing separator) or {@code null} if no parent remains. + * + * @see java.nio.file.Path#getParent() + */ + @Nullable + public UnixPath getParent() { + if (path.isEmpty() || isRoot()) { + return null; + } + int index = hasTrailingSeparator() + ? path.lastIndexOf(SEPARATOR, path.length() - 2) + : path.lastIndexOf(SEPARATOR); + if (index == -1) { + return isAbsolute() ? ROOT_PATH : null; + } else { + return new UnixPath(permitEmptyComponents, path.substring(0, index + 1)); + } + } + + /** + * Returns root component if an absolute path, otherwise {@code null}. + * + * @see java.nio.file.Path#getRoot() + */ + @Nullable + public UnixPath getRoot() { + return isAbsolute() ? ROOT_PATH : null; + } + + /** + * Returns specified range of sub-components in path joined together. + * + * @see java.nio.file.Path#subpath(int, int) + */ + public UnixPath subpath(int beginIndex, int endIndex) { + if (path.isEmpty() && beginIndex == 0 && endIndex == 1) { + return this; + } + checkArgument(beginIndex >= 0 && endIndex > beginIndex); + List subList; + try { + subList = getParts().subList(beginIndex, endIndex); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + return new UnixPath(permitEmptyComponents, JOINER.join(subList)); + } + + /** + * Returns number of components in {@code path}. + * + * @see java.nio.file.Path#getNameCount() + */ + public int getNameCount() { + if (path.isEmpty()) { + return 1; + } else if (isRoot()) { + return 0; + } else { + return getParts().size(); + } + } + + /** + * Returns component in {@code path} at {@code index}. + * + * @see java.nio.file.Path#getName(int) + */ + public UnixPath getName(int index) { + if (path.isEmpty()) { + return this; + } + try { + return new UnixPath(permitEmptyComponents, getParts().get(index)); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + } + + /** + * Returns path without extra separators or {@code .} and {@code ..}, preserving trailing slash. + * + * @see java.nio.file.Path#normalize() + */ + public UnixPath normalize() { + List parts = new ArrayList<>(); + boolean mutated = false; + int resultLength = 0; + int mark = 0; + int index; + do { + index = path.indexOf(SEPARATOR, mark); + String part = path.substring(mark, index == -1 ? path.length() : index + 1); + switch (part) { + case CURRENT_DIR: + case CURRENT_DIR + SEPARATOR: + mutated = true; + break; + case PARENT_DIR: + case PARENT_DIR + SEPARATOR: + mutated = true; + if (!parts.isEmpty()) { + resultLength -= parts.remove(parts.size() - 1).length(); + } + break; + default: + if (index != mark || index == 0) { + parts.add(part); + resultLength = part.length(); + } else { + mutated = true; + } + } + mark = index + 1; + } while (index != -1); + if (!mutated) { + return this; + } + StringBuilder result = new StringBuilder(resultLength); + for (String part : parts) { + result.append(part); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code other} appended to {@code path}. + * + * @see java.nio.file.Path#resolve(java.nio.file.Path) + */ + public UnixPath resolve(UnixPath other) { + if (other.path.isEmpty()) { + return this; + } else if (other.isAbsolute()) { + return other; + } else if (hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path + other.path); + } else { + return new UnixPath(permitEmptyComponents, path + SEPARATOR + other.path); + } + } + + /** + * Returns {@code other} resolved against parent of {@code path}. + * + * @see java.nio.file.Path#resolveSibling(java.nio.file.Path) + */ + public UnixPath resolveSibling(UnixPath other) { + checkNotNull(other); + UnixPath parent = getParent(); + return parent == null ? other : parent.resolve(other); + } + + /** + * Returns {@code other} made relative to {@code path}. + * + * @see java.nio.file.Path#relativize(java.nio.file.Path) + */ + public UnixPath relativize(UnixPath other) { + checkArgument(isAbsolute() == other.isAbsolute(), "'other' is different type of Path"); + if (path.isEmpty()) { + return other; + } + PeekingIterator left = Iterators.peekingIterator(split()); + PeekingIterator right = Iterators.peekingIterator(other.split()); + while (left.hasNext() && right.hasNext()) { + if (!left.peek().equals(right.peek())) { + break; + } + left.next(); + right.next(); + } + StringBuilder result = new StringBuilder(path.length() + other.path.length()); + while (left.hasNext()) { + result.append(PARENT_DIR); + result.append(SEPARATOR); + left.next(); + } + while (right.hasNext()) { + result.append(right.next()); + result.append(SEPARATOR); + } + if (result.length() > 0 && !other.hasTrailingSeparator()) { + result.deleteCharAt(result.length() - 1); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code true} if {@code path} starts with {@code other}. + * + * @see java.nio.file.Path#startsWith(java.nio.file.Path) + */ + public boolean startsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (me.isAbsolute() != other.isAbsolute()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } + return startsWith(split(), other.split()); + } + + private static boolean startsWith(Iterator lefts, Iterator rights) { + while (rights.hasNext()) { + if (!lefts.hasNext() || !rights.next().equals(lefts.next())) { + return false; + } + } + return true; + } + + /** + * Returns {@code true} if {@code path} ends with {@code other}. + * + * @see java.nio.file.Path#endsWith(java.nio.file.Path) + */ + public boolean endsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } else if (other.isAbsolute()) { + return me.isAbsolute() && me.path.equals(other.path); + } + return startsWith(me.splitReverse(), other.splitReverse()); + } + + /** + * Compares two paths lexicographically for ordering. + * + * @see java.nio.file.Path#compareTo(java.nio.file.Path) + */ + public int compareTo(UnixPath other) { + return ORDERING.compare(getParts(), other.getParts()); + } + + /** Converts relative path to an absolute path. */ + public UnixPath toAbsolutePath(UnixPath currentWorkingDirectory) { + checkArgument(currentWorkingDirectory.isAbsolute()); + return isAbsolute() ? this : currentWorkingDirectory.resolve(this); + } + + /** Returns {@code toAbsolutePath(ROOT_PATH)} */ + public UnixPath toAbsolutePath() { + return toAbsolutePath(ROOT_PATH); + } + + /** Removes beginning separator from path, if an absolute path. */ + public UnixPath removeBeginningSeparator() { + return isAbsolute() ? new UnixPath(permitEmptyComponents, path.substring(1)) : this; + } + + /** Adds trailing separator to path, if it isn't present. */ + public UnixPath addTrailingSeparator() { + return hasTrailingSeparator() ? this : new UnixPath(permitEmptyComponents, path + SEPARATOR); + } + + /** Removes trailing separator from path, unless it's root. */ + public UnixPath removeTrailingSeparator() { + if (!isRoot() && hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path.substring(0, path.length() - 1)); + } else { + return this; + } + } + + /** Splits path into components, excluding separators and empty strings. */ + public Iterator split() { + return getParts().iterator(); + } + + /** Splits path into components in reverse, excluding separators and empty strings. */ + public Iterator splitReverse() { + return Lists.reverse(getParts()).iterator(); + } + + @Override + public boolean equals(@Nullable Object other) { + return this == other + || other instanceof UnixPath + && path.equals(((UnixPath) other).path); + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + /** Returns path as a string. */ + @Override + public String toString() { + return path; + } + + @Override + public int length() { + return path.length(); + } + + @Override + public char charAt(int index) { + return path.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return path.subSequence(start, end); + } + + /** Returns {@code true} if this path is an empty string. */ + public boolean isEmpty() { + return path.isEmpty(); + } + + /** Returns list of path components, excluding slashes. */ + private List getParts() { + List result = lazyStringParts; + return result != null + ? result : (lazyStringParts = path.isEmpty() || isRoot() + ? Collections.emptyList() : createParts()); + } + + private List createParts() { + if (permitEmptyComponents) { + return SPLITTER_PERMIT_EMPTY_COMPONENTS + .splitToList(path.charAt(0) == SEPARATOR ? path.substring(1) : path); + } else { + return SPLITTER.splitToList(path); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/package-info.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/package-info.java new file mode 100644 index 000000000000..e07c960b5512 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/gcloud/storage/contrib/nio/package-info.java @@ -0,0 +1,82 @@ +/** + * Java 7 nio FileSystem client library for Google Cloud Storage. + * + *

This client library allows you to easily interact with Google Cloud Storage, using Java's + * standard file system API, introduced in Java 7. + * + *

How It Works

+ * + * The simplest way to get started is with {@code Paths} and {@code Files}:
   {@code
+ *
+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   List lines = Files.readAllLines(path, StandardCharsets.UTF_8);}
+ * + *

If you want to configure the bucket per-environment, it might make more sense to use the + * {@code FileSystem} API: + *

+ *   class Foo {
+ *     static String bucket = System.getProperty(...);
+ *     static FileSystem fs = FileSystems.getFileSystem(URI.create("gs://" + bucket));
+ *     void bar() {
+ *       byte[] data = "hello kitty".getBytes(StandardCharsets.UTF_8);
+ *       Path path = fs.getPath("/object");
+ *       Files.write(path, data);
+ *       data = Files.readBytes(path);
+ *     }
+ *     void doodle() {
+ *       Path path = fs.getPath("/path/to/doodle.csv");
+ *       List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ *     }
+ *   }
+ * + *

You can also use InputStream and OutputStream for streaming: + *

+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   try (InputStream input = Files.openInputStream(path)) {
+ *     // ...
+ *   }
+ * + *

You can set various attributes using + * {@link com.google.gcloud.storage.contrib.nio.CloudStorageOptions CloudStorageOptions} static + * helpers: + *

+ *   Files.write(csvPath, csvLines, StandardCharsets.UTF_8,
+ *       withMimeType(MediaType.CSV_UTF8),
+ *       withoutCaching());
+ * + *

NOTE: Cloud Storage uses a flat namespace and therefore doesn't support real + * directories. So this library supports what's known as "pseudo-directories". Any path that + * includes a trailing slash, will be considered a directory. It will always be assumed to exist, + * without performing any I/O. This allows you to do path manipulation in the same manner as you + * would with the normal UNIX file system implementation. You can disable this feature with + * {@link com.google.gcloud.storage.contrib.nio.CloudStorageConfiguration#usePseudoDirectories()}. + * + *

Unit Testing

+ * + *

Here's a simple unit test:

+ *
+ *   class MyTest {
+ *     {@literal @}Rule
+ *     public final AppEngineRule appEngine = AppEngineRule.builder().build();
+ *
+ *     {@literal @}Test
+ *     public test_fileWrite() throws Exception {
+ *       Path path = Paths.get(URI.create("gs://bucket/traditional"));
+ *       Files.write(path, "eyebrow".getBytes(StandardCharsets.US_ASCII));
+ *       assertEquals("eyebrow", new String(Files.readBytes(path), StandardCharsets.US_ASCII));
+ *     }
+ *   }
+ * + *

Non-SPI Interface

+ * + *

If you don't want to rely on Java SPI, which requires a META-INF file in your jar generated by + * Google Auto, you can instantiate this file system directly as follows:

   {@code
+ *
+ *   CloudStorageFileSystem fs = CloudStorageFileSystemProvider.forBucket("bucket");
+ *   byte[] data = "hello kitty".getBytes(StandardCharsets.UTF_8);
+ *   Path path = fs.getPath("/object");
+ *   Files.write(path, data);
+ *   data = Files.readBytes(path);}
+ */ +@javax.annotation.ParametersAreNonnullByDefault +package com.google.gcloud.storage.contrib.nio; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java new file mode 100644 index 000000000000..090ffa04ef77 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/AppEngineRule.java @@ -0,0 +1,25 @@ +package com.google.gcloud.storage.contrib.nio; + +import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; + +import org.junit.rules.ExternalResource; + +import java.io.IOException; + +/** JUnit rule for App Engine testing environment. */ +public final class AppEngineRule extends ExternalResource { + + private LocalServiceTestHelper helper; + + @Override + protected void before() throws IOException { + helper = new LocalServiceTestHelper(new LocalDatastoreServiceTestConfig()); + helper.setUp(); + } + + @Override + protected void after() { + helper.tearDown(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfigurationTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfigurationTest.java new file mode 100644 index 000000000000..9bb987e86707 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageConfigurationTest.java @@ -0,0 +1,58 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableMap; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link CloudStorageConfiguration}. */ +@RunWith(JUnit4.class) +public class CloudStorageConfigurationTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void testBuilder() { + CloudStorageConfiguration config = CloudStorageConfiguration.builder() + .workingDirectory("/omg") + .permitEmptyPathComponents(true) + .stripPrefixSlash(false) + .usePseudoDirectories(false) + .blockSize(666) + .build(); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + } + + @Test + public void testFromMap() { + CloudStorageConfiguration config = CloudStorageConfiguration.fromMap( + new ImmutableMap.Builder() + .put("workingDirectory", "/omg") + .put("permitEmptyPathComponents", true) + .put("stripPrefixSlash", false) + .put("usePseudoDirectories", false) + .put("blockSize", 666) + .build()); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + } + + @Test + public void testFromMap_badKey_throwsIae() { + thrown.expect(IllegalArgumentException.class); + CloudStorageConfiguration.fromMap(ImmutableMap.of("lol", "/omg")); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java new file mode 100644 index 000000000000..c376d2456dfc --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java @@ -0,0 +1,100 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; + +/** Unit tests for {@link CloudStorageFileAttributeView}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributeViewTest { + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + private Path path; + + @Before + public void before() throws Exception { + path = Paths.get(URI.create("gs://red/water")); + } + + @Test + public void testReadAttributes() throws Exception { + Files.write(path, HAPPY, withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes().cacheControl().get()).isEqualTo("potato"); + } + + @Test + public void testReadAttributes_notFound_throwsNoSuchFileException() throws Exception { + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + thrown.expect(NoSuchFileException.class); + lazyAttributes.readAttributes(); + } + + @Test + public void testReadAttributes_pseudoDirectory() throws Exception { + Path dir = Paths.get(URI.create("gs://red/rum/")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(dir, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes()) + .isInstanceOf(CloudStoragePseudoDirectoryAttributes.class); + } + + @Test + public void testName() throws Exception { + Files.write(path, HAPPY, withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.name()).isEqualTo("gcs"); + } + + @Test + public void testEquals_equalsTester() throws Exception { + new EqualsTester() + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), + CloudStorageFileAttributeView.class), + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), + CloudStorageFileAttributeView.class)) + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/lol/dog")), + CloudStorageFileAttributeView.class)) + .testEquals(); + } + + @Test + public void testNullness() throws Exception { + new NullPointerTester() + .setDefault(FileTime.class, FileTime.fromMillis(0)) + .testAllPublicInstanceMethods( + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class)); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java new file mode 100644 index 000000000000..856a5dcb2c8c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileAttributesTest.java @@ -0,0 +1,143 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withAcl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentDisposition; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentEncoding; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withMimeType; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withUserMetadata; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** Unit tests for {@link CloudStorageFileAttributes}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributesTest { + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + private Path path; + private Path dir; + + @Before + public void before() throws Exception { + path = Paths.get(URI.create("gs://bucket/randompath")); + dir = Paths.get(URI.create("gs://bucket/randompath/")); + } + + @Test + public void testCacheControl() throws Exception { + Files.write(path, HAPPY, withCacheControl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); + } + + @Test + public void testMimeType() throws Exception { + Files.write(path, HAPPY, withMimeType("text/potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/potato"); + } + + @Test + public void testAcl() throws Exception { + Files.write(path, HAPPY, withAcl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .isEqualTo("potato"); + } + + @Test + public void testContentDisposition() throws Exception { + Files.write(path, HAPPY, withContentDisposition("crash call")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) + .isEqualTo("crash call"); + } + + @Test + public void testContentEncoding() throws Exception { + Files.write(path, HAPPY, withContentEncoding("my content encoding")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("my content encoding"); + } + + @Test + public void testUserMetadata() throws Exception { + Files.write(path, HAPPY, withUserMetadata("green", "bean")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).userMetadata().get("green")) + .isEqualTo("bean"); + } + + @Test + public void testIsDirectory() throws Exception { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isDirectory()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isDirectory()).isTrue(); + } + + @Test + public void testIsRegularFile() throws Exception { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isRegularFile()) + .isTrue(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isRegularFile()) + .isFalse(); + } + + @Test + public void testIsOther() throws Exception { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isOther()).isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isOther()).isFalse(); + } + + @Test + public void testIsSymbolicLink() throws Exception { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + } + + @Test + public void testEquals_equalsTester() throws Exception { + Files.write(path, HAPPY, withMimeType("text/plain")); + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + Files.write(path, HAPPY, withMimeType("text/potato")); + CloudStorageFileAttributes b1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + new EqualsTester().addEqualityGroup(a1, a2).addEqualityGroup(b1, b2).testEquals(); + } + + @Test + public void testNullness() throws Exception { + Files.write(path, HAPPY); + CloudStorageFileAttributes pathAttributes = + Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes dirAttributes = + Files.readAttributes(dir, CloudStorageFileAttributes.class); + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicInstanceMethods(pathAttributes); + tester.testAllPublicInstanceMethods(dirAttributes); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java new file mode 100644 index 000000000000..8dad56ff984e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java @@ -0,0 +1,592 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.forBucket; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withMimeType; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; +import static org.junit.Assume.assumeTrue; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.List; + +/** Unit tests for {@link CloudStorageFileSystemProvider}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemProviderTest { + + private static final List FILE_CONTENTS = ImmutableList.of( + "To be, or not to be, that is the question—", + "Whether 'tis Nobler in the mind to suffer", + "The Slings and Arrows of outrageous Fortune,", + "Or to take Arms against a Sea of troubles,", + "And by opposing, end them? To die, to sleep—", + "No more; and by a sleep, to say we end", + "The Heart-ache, and the thousand Natural shocks", + "That Flesh is heir to? 'Tis a consummation"); + + private static final String SINGULARITY = "A string"; + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + @Test + public void testSize() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(Files.size(path)).isEqualTo(SINGULARITY.getBytes(UTF_8).length); + } + + @Test + public void testSize_trailingSlash_returnsFakePseudoDirectorySize() throws Exception { + assertThat(Files.size(Paths.get(URI.create("gs://bucket/wat/")))).isEqualTo(1); + } + + @Test + public void testSize_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + byte[] rapture = SINGULARITY.getBytes(UTF_8); + Files.write(path, rapture); + assertThat(Files.size(path)).isEqualTo(rapture.length); + } + } + + @Test + public void testReadAllBytes() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testReadAllBytes_trailingSlash() throws Exception { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.readAllBytes(Paths.get(URI.create("gs://bucket/wat/"))); + } + + @Test + public void testNewByteChannelRead() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + byte[] data = SINGULARITY.getBytes(UTF_8); + Files.write(path, data); + try (ReadableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(data.length); + assertThat(input.read(buffer)).isEqualTo(data.length); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo(SINGULARITY); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + } + } + + @Test + public void testNewByteChannelRead_seeking() throws Exception { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "helloworld".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(5); + assertThat(input.position()).isEqualTo(5); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(10); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("world"); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(0); + assertThat(input.position()).isEqualTo(0); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(5); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("hello"); + } + } + + @Test + public void testNewByteChannelRead_seekBeyondSize_reportsEofOnNextRead() throws Exception { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "hellocat".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(10); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(11); + assertThat(input.read(buffer)).isEqualTo(-1); + assertThat(input.size()).isEqualTo(8); + } + } + + @Test + public void testNewByteChannelRead_trailingSlash() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.newByteChannel(path); + } + + @Test + public void testNewByteChannelRead_notFound() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wednesday")); + thrown.expect(NoSuchFileException.class); + Files.newByteChannel(path); + } + + @Test + public void testNewByteChannelWrite() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/tests")); + try (SeekableByteChannel output = Files.newByteChannel(path, WRITE)) { + assertThat(output.position()).isEqualTo(0); + assertThat(output.size()).isEqualTo(0); + ByteBuffer buffer = ByteBuffer.wrap("filec".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(5); + assertThat(output.size()).isEqualTo(5); + buffer = ByteBuffer.wrap("onten".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(10); + assertThat(output.size()).isEqualTo(10); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo("fileconten"); + } + + @Test + public void testNewInputStream() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (InputStream input = Files.newInputStream(path)) { + byte[] data = new byte[SINGULARITY.getBytes(UTF_8).length]; + input.read(data); + assertThat(new String(data, UTF_8)).isEqualTo(SINGULARITY); + } + } + + @Test + public void testNewInputStream_trailingSlash() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + } + } + + @Test + public void testNewInputStream_notFound() throws Exception { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + thrown.expect(NoSuchFileException.class); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + } + } + + @Test + public void testNewOutputStream() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateByDefault() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateExplicitly() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path, TRUNCATE_EXISTING)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_trailingSlash() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + try (OutputStream output = Files.newOutputStream(path)) { + } + } + + @Test + public void testNewOutputStream_createNew() throws Exception { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + try (OutputStream output = Files.newOutputStream(path, CREATE_NEW)) { + } + } + + @Test + public void testNewOutputStream_createNew_alreadyExists() throws Exception { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + thrown.expect(FileAlreadyExistsException.class); + try (OutputStream output = Files.newOutputStream(path, CREATE_NEW)) { + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_throwsIae() throws Exception { + Path path = Paths.get(URI.create("gs://double/slash//yep")); + thrown.expect(IllegalArgumentException.class); + Files.write(path, FILE_CONTENTS, UTF_8); + } + + @Test + public void testWrite_objectNameWithExtraSlashes_canBeNormalized() throws Exception { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(false))) { + Path path = fs.getPath("adipose//yep").normalize(); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + GcsService gcsService = GcsServiceFactory.createGcsService(); + assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose/yep"))).isNotNull(); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_permitEmptyPathComponents() throws Exception { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(true))) { + Path path = fs.getPath("adipose//yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + GcsService gcsService = GcsServiceFactory.createGcsService(); + assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose//yep"))).isNotNull(); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite_absoluteObjectName_prefixSlashGetsRemoved() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose/yep")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + GcsService gcsService = GcsServiceFactory.createGcsService(); + assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "adipose/yep"))).isNotNull(); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testWrite_absoluteObjectName_disableStrip_slashGetsPreserved() throws Exception { + try (CloudStorageFileSystem fs = + forBucket("greenbean", + CloudStorageConfiguration.builder() + .stripPrefixSlash(false) + .build())) { + Path path = fs.getPath("/adipose/yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + GcsService gcsService = GcsServiceFactory.createGcsService(); + assertThat(gcsService.getMetadata(new GcsFilename("greenbean", "/adipose/yep"))).isNotNull(); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + } + + @Test + public void testWrite_trailingSlash() throws Exception { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.write(Paths.get(URI.create("gs://greenbean/adipose/")), FILE_CONTENTS, UTF_8); + } + + @Test + public void testExists() throws Exception { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isFalse(); + Files.write(Paths.get(URI.create("gs://military/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isTrue(); + } + + @Test + public void testExists_trailingSlash() throws Exception { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/.")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/..")))).isTrue(); + } + + @Test + public void testExists_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("military", usePseudoDirectories(false))) { + assertThat(Files.exists(fs.getPath("fashion/"))).isFalse(); + } + } + + @Test + public void testDelete() throws Exception { + Files.write(Paths.get(URI.create("gs://love/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isTrue(); + Files.delete(Paths.get(URI.create("gs://love/fashion"))); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isFalse(); + } + + @Test + public void testDelete_dotDirNotNormalized_throwsIae() throws Exception { + thrown.expect(IllegalArgumentException.class); + Files.delete(Paths.get(URI.create("gs://love/fly/../passion"))); + } + + @Test + public void testDelete_trailingSlash() throws Exception { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.delete(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testDelete_trailingSlash_disablePseudoDirectories() throws Exception { + try (CloudStorageFileSystem fs = forBucket("pumpkin", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + Files.write(path, FILE_CONTENTS, UTF_8); + GcsService gcsService = GcsServiceFactory.createGcsService(); + assertThat(gcsService.getMetadata(new GcsFilename("pumpkin", "wat/"))).isNotNull(); + Files.delete(path); + assertThat(gcsService.getMetadata(new GcsFilename("pumpkin", "wat/"))).isNull(); + } + } + + @Test + public void testDelete_notFound() throws Exception { + GcsService gcsService = GcsServiceFactory.createGcsService(); + assumeTrue(!gcsService.delete(new GcsFilename("loveh", "passionehu"))); // XXX: b/15832793 + thrown.expect(NoSuchFileException.class); + Files.delete(Paths.get(URI.create("gs://loveh/passionehu"))); + } + + @Test + public void testDeleteIfExists() throws Exception { + GcsService gcsService = GcsServiceFactory.createGcsService(); + assumeTrue(!gcsService.delete(new GcsFilename("loveh", "passionehu"))); // XXX: b/15832793 + assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isFalse(); + Files.write(Paths.get(URI.create("gs://love/passionz")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isTrue(); + } + + @Test + public void testDeleteIfExists_trailingSlash() throws Exception { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.deleteIfExists(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testCopy() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target); + assertThat(new String(Files.readAllBytes(target), UTF_8)).isEqualTo("(✿◕ ‿◕ )ノ"); + assertThat(Files.exists(source)).isTrue(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testCopy_sourceMissing_throwsNoSuchFileException() throws Exception { + thrown.expect(NoSuchFileException.class); + Files.copy( + Paths.get(URI.create("gs://military/fashion.show")), + Paths.get(URI.create("gs://greenbean/adipose"))); + } + + @Test + public void testCopy_targetExists_throwsFileAlreadyExistsException() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(FileAlreadyExistsException.class); + Files.copy(source, target); + } + + @Test + public void testCopyReplace_targetExists_works() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target, REPLACE_EXISTING); + } + + @Test + public void testCopy_directory_doesNothing() throws Exception { + Path source = Paths.get(URI.create("gs://military/fundir/")); + Path target = Paths.get(URI.create("gs://greenbean/loldir/")); + Files.copy(source, target); + } + + @Test + public void testCopy_atomic_throwsUnsupported() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(UnsupportedOperationException.class); + Files.copy(source, target, ATOMIC_MOVE); + } + + @Test + public void testMove() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.move(source, target); + assertThat(new String(Files.readAllBytes(target), UTF_8)).isEqualTo("(✿◕ ‿◕ )ノ"); + assertThat(Files.exists(source)).isFalse(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testCreateDirectory() throws Exception { + Path path = Paths.get(URI.create("gs://greenbean/dir/")); + Files.createDirectory(path); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testMove_atomicMove_notSupported() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(AtomicMoveNotSupportedException.class); + Files.move(source, target, ATOMIC_MOVE); + } + + @Test + public void testIsDirectory() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://doodle"))) { + assertThat(Files.isDirectory(fs.getPath(""))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("./"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/../"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/../"))).isTrue(); + } + } + + @Test + public void testIsDirectory_trailingSlash_alwaysTrue() throws Exception { + assertThat(Files.isDirectory(Paths.get(URI.create("gs://military/fundir/")))).isTrue(); + } + + @Test + public void testIsDirectory_trailingSlash_pseudoDirectoriesDisabled_false() throws Exception { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + assertThat(Files.isDirectory(fs.getPath("fundir/"))).isFalse(); + } + } + + @Test + public void testCopy_withCopyAttributes_preservesAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withMimeType("text/lolcat"), + withCacheControl("public; max-age=666")); + Files.copy(source, target, COPY_ATTRIBUTES); + GcsService gcsService = GcsServiceFactory.createGcsService(); + GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); + assertThat(metadata.getOptions().getMimeType()).isEqualTo("text/lolcat"); + assertThat(metadata.getOptions().getCacheControl()).isEqualTo("public; max-age=666"); + } + + @Test + public void testCopy_withoutOptions_doesntPreservesAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withMimeType("text/lolcat"), + withCacheControl("public; max-age=666")); + Files.copy(source, target); + GcsService gcsService = GcsServiceFactory.createGcsService(); + GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); + assertThat(metadata.getOptions().getMimeType()).isNull(); + assertThat(metadata.getOptions().getCacheControl()).isNull(); + } + + @Test + public void testCopy_overwriteAttributes() throws Exception { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withMimeType("text/lolcat"), + withCacheControl("public; max-age=666")); + Files.copy(source, target, COPY_ATTRIBUTES, + withMimeType("text/palfun")); + GcsService gcsService = GcsServiceFactory.createGcsService(); + GcsFileMetadata metadata = gcsService.getMetadata(new GcsFilename("greenbean", "adipose")); + assertThat(metadata.getOptions().getMimeType()).isEqualTo("text/palfun"); + assertThat(metadata.getOptions().getCacheControl()).isEqualTo("public; max-age=666"); + } + + @Test + public void testNullness() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://blood"))) { + NullPointerTester tester = new NullPointerTester() + .setDefault(URI.class, URI.create("gs://blood")) + .setDefault(Path.class, fs.getPath("and/one")) + .setDefault(OpenOption.class, StandardOpenOption.CREATE) + .setDefault(CopyOption.class, StandardCopyOption.COPY_ATTRIBUTES); + tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class); + tester.testAllPublicInstanceMethods(new CloudStorageFileSystemProvider()); + } + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder() + .permitEmptyPathComponents(value) + .build(); + } + + private static CloudStorageConfiguration usePseudoDirectories(boolean value) { + return CloudStorageConfiguration.builder() + .usePseudoDirectories(value) + .build(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java new file mode 100644 index 000000000000..7cb750238b14 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageFileSystemTest.java @@ -0,0 +1,111 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** Unit tests for {@link CloudStorageFileSystem}. */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemTest { + + private static final String ALONE = "" + + "To be, or not to be, that is the question—\n" + + "Whether 'tis Nobler in the mind to suffer\n" + + "The Slings and Arrows of outrageous Fortune,\n" + + "Or to take Arms against a Sea of troubles,\n" + + "And by opposing, end them? To die, to sleep—\n" + + "No more; and by a sleep, to say we end\n" + + "The Heart-ache, and the thousand Natural shocks\n" + + "That Flesh is heir to? 'Tis a consummation\n"; + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + @Test + public void testGetPath() throws Exception { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.getPath("/angel").toString()).isEqualTo("/angel"); + assertThat(fs.getPath("/angel").toUri().toString()).isEqualTo("gs://bucket/angel"); + } + } + + @Test + public void testWrite() throws Exception { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Files.write(fs.getPath("/angel"), ALONE.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(Paths.get(URI.create("gs://bucket/angel"))), UTF_8)) + .isEqualTo(ALONE); + } + + @Test + public void testRead() throws Exception { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(new String(Files.readAllBytes(fs.getPath("/angel")), UTF_8)).isEqualTo(ALONE); + } + } + + @Test + public void testExists_false() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + assertThat(Files.exists(fs.getPath("/angel"))).isFalse(); + } + } + + @Test + public void testExists_true() throws Exception { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(Files.exists(fs.getPath("/angel"))).isTrue(); + } + } + + @Test + public void testGetters() throws Exception { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.isOpen()).isTrue(); + assertThat(fs.isReadOnly()).isFalse(); + assertThat(fs.getRootDirectories()).containsExactly(fs.getPath("/")); + assertThat(fs.getFileStores()).isEmpty(); + assertThat(fs.getSeparator()).isEqualTo("/"); + assertThat(fs.supportedFileAttributeViews()).containsExactly("basic", "gcs"); + } + } + + @Test + public void testEquals() throws Exception { + try (FileSystem bucket1 = CloudStorageFileSystem.forBucket("bucket"); + FileSystem bucket2 = FileSystems.getFileSystem(URI.create("gs://bucket")); + FileSystem doge1 = CloudStorageFileSystem.forBucket("doge"); + FileSystem doge2 = FileSystems.getFileSystem(URI.create("gs://doge"))) { + new EqualsTester() + .addEqualityGroup(bucket1, bucket2) + .addEqualityGroup(doge1, doge2) + .testEquals(); + } + } + + @Test + public void testNullness() throws Exception { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + NullPointerTester tester = new NullPointerTester() + .setDefault(CloudStorageConfiguration.class, CloudStorageConfiguration.DEFAULT); + tester.testAllPublicStaticMethods(CloudStorageFileSystem.class); + tester.testAllPublicInstanceMethods(fs); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java new file mode 100644 index 000000000000..0ed9dc5f7507 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageOptionsTest.java @@ -0,0 +1,105 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; +import static com.google.common.truth.Truth.assertThat; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withAcl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withCacheControl; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentDisposition; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withContentEncoding; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withMimeType; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withUserMetadata; +import static com.google.gcloud.storage.contrib.nio.CloudStorageOptions.withoutCaching; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** Unit tests for {@link CloudStorageOptions}. */ +@RunWith(JUnit4.class) +public class CloudStorageOptionsTest { + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + @Test + public void testWithoutCaching() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withoutCaching()); + assertThat(getMetadata("bucket", "path").getOptions().getCacheControl()).isEqualTo("no-cache"); + } + + @Test + public void testCacheControl() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withCacheControl("potato")); + assertThat(getMetadata("bucket", "path").getOptions().getCacheControl()).isEqualTo("potato"); + } + + @Test + public void testWithAcl() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withAcl("mine empire of dirt")); + assertThat(getMetadata("bucket", "path").getOptions().getAcl()) + .isEqualTo("mine empire of dirt"); + } + + @Test + public void testWithContentDisposition() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withContentDisposition("bubbly fun")); + assertThat(getMetadata("bucket", "path").getOptions().getContentDisposition()) + .isEqualTo("bubbly fun"); + } + + @Test + public void testWithContentEncoding() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withContentEncoding("gzip")); + assertThat(getMetadata("bucket", "path").getOptions().getContentEncoding()).isEqualTo("gzip"); + } + + @Test + public void testWithUserMetadata() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withUserMetadata("nolo", "contendere"), + withUserMetadata("eternal", "sadness")); + GcsFileMetadata metadata = getMetadata("bucket", "path"); + assertThat(metadata.getOptions().getUserMetadata().get("nolo")).isEqualTo("contendere"); + assertThat(metadata.getOptions().getUserMetadata().get("eternal")).isEqualTo("sadness"); + } + + @Test + public void testWithMimeType_string() throws Exception { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + withMimeType("text/plain")); + assertThat(getMetadata("bucket", "path").getOptions().getMimeType()).isEqualTo("text/plain"); + } + + private static GcsFileMetadata getMetadata(String bucket, String objectName) throws Exception { + return createGcsService().getMetadata(new GcsFilename(bucket, objectName)); + } + + @Test + public void testNullness() throws Exception { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicStaticMethods(CloudStorageOptions.class); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java new file mode 100644 index 000000000000..aaf9e5b4fd16 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStoragePathTest.java @@ -0,0 +1,486 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gcloud.storage.contrib.nio.CloudStorageFileSystem.forBucket; + +import com.google.common.collect.Iterables; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.ProviderMismatchException; + +/** Unit tests for {@link CloudStoragePath}. */ +@RunWith(JUnit4.class) +public class CloudStoragePathTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final AppEngineRule appEngineRule = new AppEngineRule(); + + @Test + public void testCreate_neverRemoveExtraSlashes() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("lol//cat").toString()).isEqualTo("lol//cat"); + assertThat((Object) fs.getPath("lol//cat")).isEqualTo(fs.getPath("lol//cat")); + } + } + + @Test + public void testCreate_preservesTrailingSlash() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("lol/cat/").toString()).isEqualTo("lol/cat/"); + assertThat((Object) fs.getPath("lol/cat/")).isEqualTo(fs.getPath("lol/cat/")); + } + } + + @Test + public void testGetGcsFilename_empty_notAllowed() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("").getGcsFilename(); + } + } + + @Test + public void testGetGcsFilename_stripsPrefixSlash() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi").getGcsFilename().getObjectName()).isEqualTo("hi"); + } + } + + @Test + public void testGetGcsFilename_overrideStripPrefixSlash_doesntStripPrefixSlash() { + try (CloudStorageFileSystem fs = forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("/hi").getGcsFilename().getObjectName()).isEqualTo("/hi"); + } + } + + @Test + public void testGetGcsFilename_extraSlashes_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("a//b").getGcsFilename(); + } + } + + @Test + public void testGetGcsFilename_overridepermitEmptyPathComponents() { + try (CloudStorageFileSystem fs = forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a//b").getGcsFilename().getObjectName()).isEqualTo("a//b"); + } + } + + @Test + public void testGetGcsFilename_freaksOutOnExtraSlashesAndDotDirs() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("a//b/..").getGcsFilename(); + } + } + + @Test + public void testNameCount() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/").getNameCount()).isEqualTo(0); + assertThat(fs.getPath("/hi/").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(fs.getPath("hi/yo").getNameCount()).isEqualTo(2); + } + } + + @Test + public void testGetName() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("").getName(0).toString()).isEqualTo(""); + assertThat(fs.getPath("/hi").getName(0).toString()).isEqualTo("hi"); + assertThat(fs.getPath("hi/there").getName(1).toString()).isEqualTo("there"); + } + } + + @Test + public void testGetName_negative_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("angel").getName(-1); + } + } + + @Test + public void testGetName_overflow_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("angel").getName(1); + } + } + + @Test + public void testIterator() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(Iterables.get(fs.getPath("/dog/mog"), 0).toString()).isEqualTo("dog"); + assertThat(Iterables.get(fs.getPath("/dog/mog"), 1).toString()).isEqualTo("mog"); + assertThat(Iterables.size(fs.getPath("/"))).isEqualTo(0); + assertThat(Iterables.size(fs.getPath(""))).isEqualTo(1); + assertThat(Iterables.get(fs.getPath(""), 0).toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/").normalize().toString()).isEqualTo("/"); + assertThat(fs.getPath("a/x/../b/x/..").normalize().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("/x/x/../../♡").normalize().toString()).isEqualTo("/♡"); + assertThat(fs.getPath("/x/x/./.././.././♡").normalize().toString()).isEqualTo("/♡"); + } + } + + @Test + public void testNormalize_dot_becomesBlank() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath(".").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_trailingSlash_isPreserved() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("o/").normalize().toString()).isEqualTo("o/"); + } + } + + @Test + public void testNormalize_doubleDot_becomesBlank() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("..").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath("../..").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_extraSlashes_getRemoved() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("//life///b/good//").normalize().toString()).isEqualTo("/life/b/good/"); + } + } + + @Test + public void testToRealPath_hasDotDir_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + fs.getPath("a/hi./b").toRealPath(); + fs.getPath("a/.hi/b").toRealPath(); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("dot-dir"); + fs.getPath("a/./b").toRealPath(); + } + } + + @Test + public void testToRealPath_hasDotDotDir_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + fs.getPath("a/hi../b").toRealPath(); + fs.getPath("a/..hi/b").toRealPath(); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("dot-dir"); + fs.getPath("a/../b").toRealPath(); + } + } + + @Test + public void testToRealPath_extraSlashes_throwsIae() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("extra slashes"); + fs.getPath("a//b").toRealPath(); + } + } + + @Test + public void testToRealPath_overridePermitEmptyPathComponents_extraSlashes_slashesRemain() { + try (CloudStorageFileSystem fs = forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("/life///b/./good/").toRealPath().toString()).isEqualTo("life///b/./good/"); + } + } + + @Test + public void testToRealPath_permitEmptyPathComponents_doesNotNormalize() { + try (CloudStorageFileSystem fs = forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("a"); + assertThat(fs.getPath("a//b").toRealPath().toString()).isEqualTo("a//b"); + assertThat(fs.getPath("a//./b//..").toRealPath().toString()).isEqualTo("a//./b//.."); + } + } + + @Test + public void testToRealPath_withWorkingDirectory_makesAbsolute() { + try (CloudStorageFileSystem fs = forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("lol/a"); + } + } + + @Test + public void testToRealPath_disableStripPrefixSlash_makesPathAbsolute() { + try (CloudStorageFileSystem fs = forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("/a"); + assertThat(fs.getPath("/a").toRealPath().toString()).isEqualTo("/a"); + } + } + + @Test + public void testToRealPath_trailingSlash_getsPreserved() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("a/b/").toRealPath().toString()).isEqualTo("a/b/"); + } + } + + @Test + public void testNormalize_empty_returnsEmpty() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_preserveTrailingSlash() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("a/b/../c/").normalize().toString()).isEqualTo("a/c/"); + assertThat(fs.getPath("a/b/./c/").normalize().toString()).isEqualTo("a/b/c/"); + } + } + + @Test + @SuppressWarnings("null") + public void testGetParent_preserveTrailingSlash() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("a/b/c").getParent().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("a/b/c/").getParent().toString()).isEqualTo("a/b/"); + assertThat((Object) fs.getPath("").getParent()).isNull(); + assertThat((Object) fs.getPath("/").getParent()).isNull(); + assertThat((Object) fs.getPath("aaa").getParent()).isNull(); + assertThat((Object) (fs.getPath("aaa/").getParent())).isNull(); + } + } + + @Test + @SuppressWarnings("null") + public void testGetRoot() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hello").getRoot().toString()).isEqualTo("/"); + assertThat((Object) fs.getPath("hello").getRoot()).isNull(); + } + } + + @Test + public void testRelativize() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/foo/bar/lol/cat") + .relativize(fs.getPath("/foo/a/b/../../c")).toString()) + .isEqualTo("../../../a/b/../../c"); + } + } + + @Test + public void testRelativize_providerMismatch() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + fs.getPath("/etc").relativize(Paths.get("/dog")); + } + } + + @Test + @SuppressWarnings("ReturnValueIgnored") // testing that an Exception is thrown + public void testRelativize_providerMismatch2() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + Paths.get("/dog").relativize(fs.getPath("/etc")); + } + } + + @Test + public void testResolve() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi").resolve("there").toString()).isEqualTo("/hi/there"); + assertThat(fs.getPath("hi").resolve("there").toString()).isEqualTo("hi/there"); + } + } + + @Test + public void testResolve_providerMismatch() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + fs.getPath("etc").resolve(Paths.get("/dog")); + } + } + + @Test + public void testIsAbsolute() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi").isAbsolute()).isTrue(); + assertThat(fs.getPath("hi").isAbsolute()).isFalse(); + } + } + + @Test + public void testToAbsolutePath() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat((Object) fs.getPath("/hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + assertThat((Object) fs.getPath("hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + } + } + + @Test + public void testToAbsolutePath_withWorkingDirectory() { + try (CloudStorageFileSystem fs = forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toAbsolutePath().toString()).isEqualTo("/lol/a"); + } + } + + @Test + @SuppressWarnings("null") + public void testGetFileName() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").getFileName().toString()).isEqualTo("there"); + assertThat(fs.getPath("military/fashion/show").getFileName().toString()).isEqualTo("show"); + } + } + + @Test + public void testCompareTo() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/there"))).isEqualTo(0); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therf"))).isEqualTo(-1); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therd"))).isEqualTo(1); + } + } + + @Test + public void testStartsWith() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("hi"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath(""))).isFalse(); + } + } + + @Test + public void testEndsWith() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/blag/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/there"))).isFalse(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry"))).isTrue(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry/"))).isTrue(); + assertThat(fs.getPath("/hi/there/").endsWith(fs.getPath("/"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath(""))).isFalse(); + assertThat(fs.getPath("").endsWith(fs.getPath(""))).isTrue(); + } + } + + /** @see "http://stackoverflow.com/a/10068306" */ + @Test + public void testResolve_willWorkWithRecursiveCopy() throws Exception { + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path relativeSourcePath = fsSource.getPath("file.txt"); + assertThat((Object) targetPath.resolve(relativeSourcePath)) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + /** @see "http://stackoverflow.com/a/10068306" */ + @Test + public void testRelativize_willWorkWithRecursiveCopy() throws Exception { + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path sourcePath = fsSource.getPath("/sloth/"); + Path file = fsSource.getPath("/sloth/file.txt"); + assertThat((Object) targetPath.resolve(sourcePath.relativize(file))) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + @Test + public void testToFile_unsupported() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + Path path = fs.getPath("/lol"); + thrown.expect(UnsupportedOperationException.class); + path.toFile(); + } + } + + @Test + public void testEquals() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + new EqualsTester() + // These are obviously equal. + .addEqualityGroup(fs.getPath("/hello/cat"), fs.getPath("/hello/cat")) + // These are equal because equals() runs things through toRealPath() + .addEqualityGroup(fs.getPath("great/commandment"), fs.getPath("/great/commandment")) + .addEqualityGroup(fs.getPath("great/commandment/"), fs.getPath("/great/commandment/")) + // Equals shouldn't do error checking or normalization. + .addEqualityGroup(fs.getPath("foo/../bar"), fs.getPath("foo/../bar")) + .addEqualityGroup(fs.getPath("bar")) + .testEquals(); + } + } + + @Test + public void testEquals_currentDirectoryIsTakenIntoConsideration() { + try (CloudStorageFileSystem fs = forBucket("doodle", workingDirectory("/hello"))) { + new EqualsTester() + .addEqualityGroup(fs.getPath("cat"), fs.getPath("/hello/cat")) + .addEqualityGroup(fs.getPath(""), fs.getPath("/hello")) + .testEquals(); + } + } + + @Test + public void testNullness() { + try (CloudStorageFileSystem fs = forBucket("doodle")) { + NullPointerTester tester = new NullPointerTester() + .setDefault(Path.class, fs.getPath("sup")); + tester.testAllPublicStaticMethods(CloudStoragePath.class); + tester.testAllPublicInstanceMethods(fs.getPath("sup")); + } + } + + private static CloudStorageConfiguration stripPrefixSlash(boolean value) { + return CloudStorageConfiguration.builder() + .stripPrefixSlash(value) + .build(); + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder() + .permitEmptyPathComponents(value) + .build(); + } + + private static CloudStorageConfiguration workingDirectory(String value) { + return CloudStorageConfiguration.builder() + .workingDirectory(value) + .build(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java new file mode 100644 index 000000000000..c46ec0ac7981 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -0,0 +1,146 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.appengine.tools.cloudstorage.GcsFileMetadata; +import com.google.appengine.tools.cloudstorage.GcsFileOptions; +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.appengine.tools.cloudstorage.GcsInputChannel; +import com.google.appengine.tools.cloudstorage.GcsService; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; + +/** Unit tests for {@link CloudStorageReadChannel}. */ +@RunWith(JUnit4.class) +public class CloudStorageReadChannelTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + private final GcsService gcsService = mock(GcsService.class); + private final GcsInputChannel gcsChannel = mock(GcsInputChannel.class); + private final GcsFilename file = new GcsFilename("enya", "rocks"); + private final GcsFileOptions options = GcsFileOptions.getDefaultInstance(); + private final GcsFileMetadata metadata = new GcsFileMetadata(file, options, null, 42, null); + private CloudStorageReadChannel chan; + + @Before + public void before() throws Exception { + when(gcsService.getMetadata(eq(file))).thenReturn(metadata); + when(gcsService.openReadChannel(eq(file), anyInt())).thenReturn(gcsChannel); + when(gcsChannel.isOpen()).thenReturn(true); + chan = CloudStorageReadChannel.create(gcsService, file, 0); + verify(gcsService).getMetadata(eq(file)); + verify(gcsService).openReadChannel(eq(file), eq(0L)); + } + + @Test + public void testRead() throws Exception { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))).thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.read(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + verify(gcsChannel).read(any(ByteBuffer.class)); + verify(gcsChannel, times(3)).isOpen(); + verifyNoMoreInteractions(gcsService, gcsChannel); + } + + @Test + public void testRead_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.read(ByteBuffer.allocate(1)); + } + + @Test + public void testWrite_throwsNonWritableChannelException() throws Exception { + thrown.expect(NonWritableChannelException.class); + chan.write(ByteBuffer.allocate(1)); + } + + @Test + public void testTruncate_throwsNonWritableChannelException() throws Exception { + thrown.expect(NonWritableChannelException.class); + chan.truncate(0); + } + + @Test + public void testIsOpen() throws Exception { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + verifyNoMoreInteractions(gcsService, gcsChannel); + } + + @Test + public void testSize() throws Exception { + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).isOpen(); + verifyZeroInteractions(gcsChannel); + verifyNoMoreInteractions(gcsService); + } + + @Test + public void testSize_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.size(); + } + + @Test + public void testPosition_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(); + } + + @Test + public void testSetPosition_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(0); + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws Exception { + chan.close(); + chan.close(); + chan.close(); + } + + @Test + public void testSetPosition() throws Exception { + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(42L); + chan.position(1L); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).close(); + verify(gcsChannel, times(5)).isOpen(); + verify(gcsService, times(2)).getMetadata(eq(file)); + verify(gcsService).openReadChannel(eq(file), eq(1L)); + verifyNoMoreInteractions(gcsService, gcsChannel); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java new file mode 100644 index 000000000000..7b0e298b56c0 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/CloudStorageWriteChannelTest.java @@ -0,0 +1,107 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.appengine.tools.cloudstorage.GcsOutputChannel; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; + +/** Unit tests for {@link CloudStorageWriteChannel}. */ +@RunWith(JUnit4.class) +public class CloudStorageWriteChannelTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + private final GcsOutputChannel gcsChannel = mock(GcsOutputChannel.class); + private CloudStorageWriteChannel chan = new CloudStorageWriteChannel(gcsChannel); + + @Before + public void before() throws Exception { + when(gcsChannel.isOpen()).thenReturn(true); + } + + @Test + public void testRead_throwsNonReadableChannelException() throws Exception { + thrown.expect(NonReadableChannelException.class); + chan.read(ByteBuffer.allocate(1)); + } + + @Test + public void testWrite() throws Exception { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 'B'); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(0L); + when(gcsChannel.write(eq(buffer))).thenReturn(1); + assertThat(chan.write(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(1L); + verify(gcsChannel).write(any(ByteBuffer.class)); + verify(gcsChannel, times(5)).isOpen(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testWrite_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.write(ByteBuffer.allocate(1)); + } + + @Test + public void testIsOpen() throws Exception { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testSize() throws Exception { + assertThat(chan.size()).isEqualTo(0L); + verify(gcsChannel).isOpen(); + verifyZeroInteractions(gcsChannel); + } + + @Test + public void testSize_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.size(); + } + + @Test + public void testPosition_whenClosed_throwsCce() throws Exception { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(); + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws Exception { + chan.close(); + chan.close(); + chan.close(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/UnixPathTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/UnixPathTest.java new file mode 100644 index 000000000000..e6417fc75c8c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/gcloud/storage/contrib/nio/UnixPathTest.java @@ -0,0 +1,387 @@ +package com.google.gcloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link UnixPath}. */ +@RunWith(JUnit4.class) +public class UnixPathTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void testNormalize() { + assertThat(p(".").normalize()).isEqualTo(p("")); + assertThat(p("/").normalize()).isEqualTo(p("/")); + assertThat(p("/.").normalize()).isEqualTo(p("/")); + assertThat(p("/a/b/../c").normalize()).isEqualTo(p("/a/c")); + assertThat(p("/a/b/./c").normalize()).isEqualTo(p("/a/b/c")); + assertThat(p("a/b/../c").normalize()).isEqualTo(p("a/c")); + assertThat(p("a/b/./c").normalize()).isEqualTo(p("a/b/c")); + assertThat(p("/a/b/../../c").normalize()).isEqualTo(p("/c")); + assertThat(p("/a/b/./.././.././c").normalize()).isEqualTo(p("/c")); + } + + @Test + public void testNormalize_empty_returnsEmpty() { + assertThat(p("").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_underflow_isAllowed() { + assertThat(p("../").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_extraSlashes_getRemoved() { + assertThat(p("///").normalize()).isEqualTo(p("/")); + assertThat(p("/hi//there").normalize()).isEqualTo(p("/hi/there")); + assertThat(p("/hi////.///there").normalize()).isEqualTo(p("/hi/there")); + } + + @Test + public void testNormalize_trailingSlash() { + assertThat(p("/hi/there/").normalize()).isEqualTo(p("/hi/there/")); + assertThat(p("/hi/there/../").normalize()).isEqualTo(p("/hi/")); + assertThat(p("/hi/there/..").normalize()).isEqualTo(p("/hi/")); + assertThat(p("hi/../").normalize()).isEqualTo(p("")); + assertThat(p("/hi/../").normalize()).isEqualTo(p("/")); + assertThat(p("hi/..").normalize()).isEqualTo(p("")); + assertThat(p("/hi/..").normalize()).isEqualTo(p("/")); + } + + @Test + public void testNormalize_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.normalize()).isSameAs(path); + path = p("/hi/there/"); + assertThat(path.normalize()).isSameAs(path); + } + + @Test + public void testResolve() { + assertThat(p("/hello").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("/hello/").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("hello/").resolve(p("cat"))).isEqualTo(p("hello/cat")); + assertThat(p("hello/").resolve(p("cat/"))).isEqualTo(p("hello/cat/")); + assertThat(p("hello/").resolve(p(""))).isEqualTo(p("hello/")); + assertThat(p("hello/").resolve(p("/hi/there"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolve_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.resolve(p(""))).isSameAs(path); + assertThat(p("hello").resolve(path)).isSameAs(path); + } + + @Test + public void testGetPath() { + assertThat(UnixPath.getPath(false, "hello")).isEqualTo(p("hello")); + assertThat(UnixPath.getPath(false, "hello", "cat")).isEqualTo(p("hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat")).isEqualTo(p("/hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat", "inc.")) + .isEqualTo(p("/hello/cat/inc.")); + assertThat(UnixPath.getPath(false, "hello/", "/hi/there")).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolveSibling() { + assertThat(p("/hello/cat").resolveSibling(p("dog"))).isEqualTo(p("/hello/dog")); + assertThat(p("/").resolveSibling(p("dog"))).isEqualTo(p("dog")); + } + + @Test + public void testResolveSibling_preservesTrailingSlash() { + assertThat(p("/hello/cat").resolveSibling(p("dog/"))).isEqualTo(p("/hello/dog/")); + assertThat(p("/").resolveSibling(p("dog/"))).isEqualTo(p("dog/")); + } + + @Test + public void testRelativize() { + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top"))) + .isEqualTo(p("../../../mop/top")); + assertThat(p("/foo/bar/dog").relativize(p("/foo/mop/top"))) + .isEqualTo(p("../../mop/top")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top/../../mog"))) + .isEqualTo(p("../../../mop/top/../../mog")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog"))) + .isEqualTo(p("../../../../mog")); + assertThat(p("").relativize(p("foo/mop/top/"))).isEqualTo(p("foo/mop/top/")); + } + + @Test + public void testRelativize_absoluteMismatch_notAllowed() { + thrown.expect(IllegalArgumentException.class); + p("/a/b/").relativize(p("")); + } + + @Test + public void testRelativize_preservesTrailingSlash() { + // This behavior actually diverges from sun.nio.fs.UnixPath: + // bsh % print(Paths.get("/a/b/").relativize(Paths.get("/etc/"))); + // ../../etc + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog/"))) + .isEqualTo(p("../../../../mog/")); + assertThat(p("/a/b/").relativize(p("/etc/"))).isEqualTo(p("../../etc/")); + } + + @Test + public void testStartsWith() { + assertThat(p("/hi/there").startsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/therf"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/hi"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("hi"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("/a/b").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/a/b/").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("").startsWith(p(""))).isTrue(); + } + + @Test + public void testStartsWith_comparesComponentsIndividually() { + assertThat(p("/hello").startsWith(p("/hell"))).isFalse(); + assertThat(p("/hello").startsWith(p("/hello"))).isTrue(); + } + + @Test + public void testEndsWith() { + assertThat(p("/hi/there").endsWith(p("there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/blag/therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("/there"))).isFalse(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry"))).isTrue(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry/"))).isTrue(); + assertThat(p("/hi/there/").endsWith(p("/"))).isFalse(); + assertThat(p("/hi/there").endsWith(p(""))).isFalse(); + assertThat(p("").endsWith(p(""))).isTrue(); + } + + @Test + public void testEndsWith_comparesComponentsIndividually() { + assertThat(p("/hello").endsWith(p("lo"))).isFalse(); + assertThat(p("/hello").endsWith(p("hello"))).isTrue(); + } + + @Test + public void testGetParent() { + assertThat(p("").getParent()).isNull(); + assertThat(p("/").getParent()).isNull(); + assertThat(p("aaa/").getParent()).isNull(); + assertThat(p("aaa").getParent()).isNull(); + assertThat(p("/aaa/").getParent()).isEqualTo(p("/")); + assertThat(p("a/b/c").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/c/").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/").getParent()).isEqualTo(p("a/")); + } + + @Test + public void testGetRoot() { + assertThat(p("/hello").getRoot()).isEqualTo(p("/")); + assertThat(p("hello").getRoot()).isNull(); + } + + @Test + public void testGetFileName() { + assertThat(p("").getFileName()).isEqualTo(p("")); + assertThat(p("/").getFileName()).isNull(); + assertThat(p("/dark").getFileName()).isEqualTo(p("dark")); + assertThat(p("/angels/").getFileName()).isEqualTo(p("angels")); + } + + @Test + public void testEquals() { + assertThat(p("/a/").equals(p("/a/"))).isTrue(); + assertThat(p("/a/").equals(p("/b/"))).isFalse(); + assertThat(p("/b/").equals(p("/b"))).isFalse(); + assertThat(p("/b").equals(p("/b/"))).isFalse(); + assertThat(p("b").equals(p("/b"))).isFalse(); + assertThat(p("b").equals(p("b"))).isTrue(); + } + + @Test + public void testSplit() { + assertThat(p("").split().hasNext()).isFalse(); + assertThat(p("hi/there").split().hasNext()).isTrue(); + assertThat(p(p("hi/there").split().next())).isEqualTo(p("hi")); + } + + @Test + public void testToAbsolute() { + assertThat(p("lol").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol")); + assertThat(p("lol/cat").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol/cat")); + } + + @Test + public void testToAbsolute_withCurrentDirectory() { + assertThat(p("cat").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat")); + assertThat(p("cat").toAbsolutePath(p("/lol/"))).isEqualTo(p("/lol/cat")); + assertThat(p("/hi/there").toAbsolutePath(p("/lol"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testToAbsolute_preservesTrailingSlash() { + assertThat(p("cat/").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat/")); + } + + @Test + public void testSubpath() { + assertThat(p("/eins/zwei/drei/vier").subpath(0, 1)).isEqualTo(p("eins")); + assertThat(p("/eins/zwei/drei/vier").subpath(0, 2)).isEqualTo(p("eins/zwei")); + assertThat(p("eins/zwei/drei/vier/").subpath(1, 4)).isEqualTo(p("zwei/drei/vier")); + assertThat(p("eins/zwei/drei/vier/").subpath(2, 4)).isEqualTo(p("drei/vier")); + } + + @Test + public void testSubpath_empty_returnsEmpty() { + assertThat(p("").subpath(0, 1)).isEqualTo(p("")); + } + + @Test + public void testSubpath_root_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/").subpath(0, 1); + } + + @Test + public void testSubpath_negativeIndex_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(-1, 1); + } + + @Test + public void testSubpath_notEnoughElements_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(0, 5); + } + + @Test + public void testSubpath_beginAboveEnd_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(1, 0); + } + + @Test + public void testSubpath_beginAndEndEqual_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(0, 0); + } + + @Test + public void testNameCount() { + assertThat(p("").getNameCount()).isEqualTo(1); + assertThat(p("/").getNameCount()).isEqualTo(0); + assertThat(p("/hi/").getNameCount()).isEqualTo(1); + assertThat(p("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(p("hi/yo").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("hi//yo").getNameCount()).isEqualTo(2); + assertThat(p("//hi//yo//").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_permitEmptyComponents_emptiesGetCounted() { + assertThat(pp("hi//yo").getNameCount()).isEqualTo(3); + assertThat(pp("hi//yo/").getNameCount()).isEqualTo(4); + assertThat(pp("hi//yo//").getNameCount()).isEqualTo(5); + } + + @Test + public void testNameCount_permitEmptyComponents_rootComponentDoesntCount() { + assertThat(pp("hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("//hi/yo").getNameCount()).isEqualTo(3); + } + + @Test + public void testGetName() { + assertThat(p("").getName(0)).isEqualTo(p("")); + assertThat(p("/hi").getName(0)).isEqualTo(p("hi")); + assertThat(p("hi/there").getName(1)).isEqualTo(p("there")); + } + + @Test + public void testCompareTo() { + assertThat(p("/hi/there").compareTo(p("/hi/there"))).isEqualTo(0); + assertThat(p("/hi/there").compareTo(p("/hi/therf"))).isEqualTo(-1); + assertThat(p("/hi/there").compareTo(p("/hi/therd"))).isEqualTo(1); + } + + @Test + public void testCompareTo_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("a/b").compareTo(p("a//b"))).isEqualTo(0); + } + + @Test + public void testCompareTo_permitEmptyComponents_behaviorChanges() { + assertThat(p("a/b").compareTo(pp("a//b"))).isEqualTo(1); + assertThat(pp("a/b").compareTo(pp("a//b"))).isEqualTo(1); + } + + @Test + public void testCompareTo_comparesComponentsIndividually() { + assumeTrue('.' < '/'); + assertThat("hi./there".compareTo("hi/there")).isEqualTo(-1); + assertThat("hi.".compareTo("hi")).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assumeTrue('0' > '/'); + assertThat("hi0/there".compareTo("hi/there")).isEqualTo(1); + assertThat("hi0".compareTo("hi")).isEqualTo(1); + assertThat(p("hi0/there").compareTo(p("hi/there"))).isEqualTo(1); + } + + @Test + public void testSeemsLikeADirectory() { + assertThat(p("a").seemsLikeADirectory()).isFalse(); + assertThat(p("a.").seemsLikeADirectory()).isFalse(); + assertThat(p("a..").seemsLikeADirectory()).isFalse(); + assertThat(p("").seemsLikeADirectory()).isTrue(); + assertThat(p("/").seemsLikeADirectory()).isTrue(); + assertThat(p(".").seemsLikeADirectory()).isTrue(); + assertThat(p("/.").seemsLikeADirectory()).isTrue(); + assertThat(p("..").seemsLikeADirectory()).isTrue(); + assertThat(p("/..").seemsLikeADirectory()).isTrue(); + } + + @Test + public void testEquals_equalsTester() throws Exception { + new EqualsTester() + .addEqualityGroup(p("/lol"), p("/lol")) + .addEqualityGroup(p("/lol//"), p("/lol//")) + .addEqualityGroup(p("dust")) + .testEquals(); + } + + @Test + public void testNullness() { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicStaticMethods(UnixPath.class); + tester.testAllPublicInstanceMethods(p("solo")); + } + + private static UnixPath p(String path) { + return UnixPath.getPath(false, path); + } + + private static UnixPath pp(String path) { + return UnixPath.getPath(true, path); + } +} diff --git a/gcloud-java-contrib/pom.xml b/gcloud-java-contrib/pom.xml index 5d5739781727..4b8b345e8118 100644 --- a/gcloud-java-contrib/pom.xml +++ b/gcloud-java-contrib/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.gcloud gcloud-java-contrib - jar + pom GCloud Java contributions Contains packages that provide higher-level abstraction/functionality for common gcloud-java use cases. @@ -16,6 +16,9 @@ gcloud-java-contrib + + gcloud-java-nio + ${project.groupId}