diff --git a/pom.xml b/pom.xml index 11155ee..a8d75b3 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,7 @@ UTF-8 31.0.1-jre + 3.0.5 3.18.2 2.0.2 @@ -70,6 +71,11 @@ guava ${guava.version} + + com.github.ben-manes.caffeine + caffeine + ${caffeine.version} + com.auth0 java-jwt diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 8db44ca..36f4a75 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -2,6 +2,7 @@ exports org.cryptomator.cloudaccess; exports org.cryptomator.cloudaccess.api; exports org.cryptomator.cloudaccess.api.exceptions; + exports org.cryptomator.cloudaccess.requestdecorator; requires java.xml; requires com.google.common; @@ -11,4 +12,5 @@ requires okhttp.digest; requires okio; requires com.auth0.jwt; + requires com.github.benmanes.caffeine; } \ No newline at end of file diff --git a/src/main/java/org/cryptomator/cloudaccess/CloudAccess.java b/src/main/java/org/cryptomator/cloudaccess/CloudAccess.java index 1a5baf1..fe6bcc8 100644 --- a/src/main/java/org/cryptomator/cloudaccess/CloudAccess.java +++ b/src/main/java/org/cryptomator/cloudaccess/CloudAccess.java @@ -13,6 +13,7 @@ import org.cryptomator.cloudaccess.api.exceptions.VaultVerificationFailedException; import org.cryptomator.cloudaccess.api.exceptions.VaultVersionVerificationFailedException; import org.cryptomator.cloudaccess.localfs.LocalFsCloudProvider; +import org.cryptomator.cloudaccess.requestdecorator.CloudProviderDecoratorFactory; import org.cryptomator.cloudaccess.vaultformat8.VaultFormat8ProviderDecorator; import org.cryptomator.cloudaccess.webdav.WebDavCloudProvider; import org.cryptomator.cloudaccess.webdav.WebDavCredential; @@ -59,9 +60,12 @@ public static CloudProvider vaultFormat8GCMCloudAccess(CloudProvider cloudProvid verifyVaultFormat8GCMConfig(cloudProvider, pathToVault, rawKey); - VaultFormat8ProviderDecorator provider = new VaultFormat8ProviderDecorator(cloudProvider, pathToVault.resolve("d"), cryptor); - provider.initialize(); - return new MetadataCachingProviderDecorator(provider); + var decoratedCloudProvider = new CloudProviderDecoratorFactory().get(cloudProvider, cloudProvider.cachingCapability()); + + VaultFormat8ProviderDecorator vaultFormat8Provider = new VaultFormat8ProviderDecorator(decoratedCloudProvider, pathToVault.resolve("d"), cryptor); + vaultFormat8Provider.initialize(); + + return vaultFormat8Provider; } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("JVM doesn't supply a CSPRNG", e); } catch (InterruptedException e) { diff --git a/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecorator.java b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecorator.java new file mode 100644 index 0000000..162f2f4 --- /dev/null +++ b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecorator.java @@ -0,0 +1,73 @@ +package org.cryptomator.cloudaccess.requestdecorator; + +import org.cryptomator.cloudaccess.api.CloudItemList; +import org.cryptomator.cloudaccess.api.CloudItemMetadata; +import org.cryptomator.cloudaccess.api.CloudPath; +import org.cryptomator.cloudaccess.api.CloudProvider; +import org.cryptomator.cloudaccess.api.ProgressListener; +import org.cryptomator.cloudaccess.api.Quota; + +import java.io.InputStream; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.CompletionStage; + +interface CloudProviderDecorator extends CloudProvider { + + CloudProvider delegate(); + + @Override + default CompletionStage itemMetadata(CloudPath node) { + return delegate().itemMetadata(node); + } + + @Override + default CompletionStage quota(CloudPath folder) { + return delegate().quota(folder); + } + + @Override + default CompletionStage list(CloudPath folder, Optional pageToken) { + return delegate().list(folder, pageToken); + } + + @Override + default CompletionStage read(CloudPath file, long offset, long count, ProgressListener progressListener) { + return delegate().read(file, offset, count, progressListener); + } + + @Override + default CompletionStage write(CloudPath file, boolean replace, InputStream data, long size, Optional lastModified, ProgressListener progressListener) { + return delegate().write(file, replace, data, size, lastModified, progressListener); + } + + @Override + default CompletionStage createFolder(CloudPath folder) { + return delegate().createFolder(folder); + } + + @Override + default CompletionStage deleteFile(CloudPath file) { + return delegate().deleteFile(file); + } + + @Override + default CompletionStage deleteFolder(CloudPath folder) { + return delegate().deleteFolder(folder); + } + + @Override + default CompletionStage move(CloudPath source, CloudPath target, boolean replace) { + return delegate().move(source, target, replace); + } + + @Override + default boolean cachingCapability() { + return delegate().cachingCapability(); + } + + @Override + default CompletionStage pollRemoteChanges() { + return delegate().pollRemoteChanges(); + } +} diff --git a/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecoratorFactory.java b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecoratorFactory.java new file mode 100644 index 0000000..79b47c8 --- /dev/null +++ b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/CloudProviderDecoratorFactory.java @@ -0,0 +1,18 @@ +package org.cryptomator.cloudaccess.requestdecorator; + +import org.cryptomator.cloudaccess.api.CloudProvider; + +/** + * Factory class to add a caching or request-deduplication decorator around an existing {@link CloudProvider}. + */ +public class CloudProviderDecoratorFactory { + + public CloudProvider get(CloudProvider cloudProvider, boolean cloudCachingCapability) { + if (cloudCachingCapability) { + var quotaCachingDecorator = new QuotaRequestCachingDecorator(cloudProvider); + return new MetadataRequestDeduplicationDecorator(quotaCachingDecorator); + } else { + return new MetadataCachingProviderDecorator(cloudProvider); + } + } +} diff --git a/src/main/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecorator.java b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecorator.java similarity index 97% rename from src/main/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecorator.java rename to src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecorator.java index a23648b..38c8ca1 100644 --- a/src/main/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecorator.java +++ b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecorator.java @@ -1,4 +1,4 @@ -package org.cryptomator.cloudaccess; +package org.cryptomator.cloudaccess.requestdecorator; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -20,7 +20,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; -public class MetadataCachingProviderDecorator implements CloudProvider { +class MetadataCachingProviderDecorator implements CloudProvider { private final static int DEFAULT_CACHE_TIMEOUT_SECONDS = 10; @@ -93,7 +93,7 @@ public CompletionStage list(CloudPath folder, Optional pa evictFromItemAndItemListCacheIncludingDescendants(folder); } else if (delegate.cachingCapability()) { evictFromItemListCache(entry); - } else if (exception == null) { + } else if (!delegate.cachingCapability() && exception == null) { evictFromItemAndItemListCacheIncludingDescendants(folder); assert cloudItemList != null; cloudItemList.getItems().forEach(metadata -> cachedItemMetadataRequests.put(metadata.getPath(), CompletableFuture.completedFuture(metadata))); diff --git a/src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecorator.java b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecorator.java new file mode 100644 index 0000000..58e3ea3 --- /dev/null +++ b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecorator.java @@ -0,0 +1,58 @@ +package org.cryptomator.cloudaccess.requestdecorator; + +import com.github.benmanes.caffeine.cache.AsyncCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.cryptomator.cloudaccess.api.CloudItemList; +import org.cryptomator.cloudaccess.api.CloudItemMetadata; +import org.cryptomator.cloudaccess.api.CloudPath; +import org.cryptomator.cloudaccess.api.CloudProvider; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletionStage; + +/** + * Decorates an existing CloudProvider by deduplicating identical itemMetadata and list-requests so that the delegate is called only once until the future is completed. + */ +class MetadataRequestDeduplicationDecorator implements CloudProviderDecorator { + + // visible for testing + final AsyncCache cachedItemMetadataRequests; + final AsyncCache cachedItemListRequests; + + private final CloudProvider delegate; + + public MetadataRequestDeduplicationDecorator(CloudProvider delegate) { + this(delegate, // + Caffeine.newBuilder().expireAfterWrite(Duration.ofSeconds(0)).buildAsync(), // + Caffeine.newBuilder().expireAfterWrite(Duration.ofSeconds(0)).buildAsync()); + } + + MetadataRequestDeduplicationDecorator( + CloudProvider delegate, // + AsyncCache cachedItemMetadataRequests, // + AsyncCache cachedItemListRequests) { + this.delegate = delegate; + this.cachedItemMetadataRequests = cachedItemMetadataRequests; + this.cachedItemListRequests = cachedItemListRequests; + } + + @Override + public CloudProvider delegate() { + return delegate; + } + + @Override + public CompletionStage itemMetadata(CloudPath node) { + return cachedItemMetadataRequests.get(node, (key, executor) -> delegate.itemMetadata(key).toCompletableFuture()); + } + + @Override + public CompletionStage list(CloudPath folder, Optional pageToken) { + var entry = new ItemListEntry(folder, pageToken); + return cachedItemListRequests.get(entry, (key, executor) -> delegate.list(key.path, key.pageToken).toCompletableFuture()); + } + + record ItemListEntry(CloudPath path, Optional pageToken) { + } +} diff --git a/src/main/java/org/cryptomator/cloudaccess/requestdecorator/QuotaRequestCachingDecorator.java b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/QuotaRequestCachingDecorator.java new file mode 100644 index 0000000..ec0d5e3 --- /dev/null +++ b/src/main/java/org/cryptomator/cloudaccess/requestdecorator/QuotaRequestCachingDecorator.java @@ -0,0 +1,51 @@ +package org.cryptomator.cloudaccess.requestdecorator; + +import com.github.benmanes.caffeine.cache.AsyncCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.cryptomator.cloudaccess.api.CloudPath; +import org.cryptomator.cloudaccess.api.CloudProvider; +import org.cryptomator.cloudaccess.api.Quota; +import org.cryptomator.cloudaccess.api.exceptions.NotFoundException; +import org.cryptomator.cloudaccess.api.exceptions.QuotaNotAvailableException; + +import java.time.Duration; +import java.util.concurrent.CompletionStage; + +/** + * Decorates an existing CloudProvider by caching quota-requests for a duration of default 10 seconds (can be set using org.cryptomator.cloudaccess.metadatacachingprovider.timeoutSeconds). + */ +class QuotaRequestCachingDecorator implements CloudProviderDecorator { + + private final static int DEFAULT_CACHE_TIMEOUT_SECONDS = 10; + + // visible for testing + final AsyncCache quotaCache; + + private final CloudProvider delegate; + + public QuotaRequestCachingDecorator(CloudProvider delegate) { + this(delegate, Caffeine + .newBuilder() + .expireAfterWrite(Duration.ofSeconds(Integer.getInteger("org.cryptomator.cloudaccess.metadatacachingprovider.timeoutSeconds", DEFAULT_CACHE_TIMEOUT_SECONDS))) + .buildAsync()); + } + + QuotaRequestCachingDecorator(CloudProvider delegate, AsyncCache quotaCache) { + this.delegate = delegate; + this.quotaCache = quotaCache; + } + + @Override + public CloudProvider delegate() { + return delegate; + } + + @Override + public CompletionStage quota(CloudPath folder) { + return quotaCache.get(folder, k -> delegate.quota(k).whenComplete((metadata, throwable) -> { + if (throwable != null && !(throwable instanceof NotFoundException) && !(throwable instanceof QuotaNotAvailableException)) { + quotaCache.synchronous().invalidate(folder); + } + }).toCompletableFuture().join()); + } +} diff --git a/src/test/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecoratorTest.java b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecoratorTest.java similarity index 99% rename from src/test/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecoratorTest.java rename to src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecoratorTest.java index ee13dc1..b04d41a 100644 --- a/src/test/java/org/cryptomator/cloudaccess/MetadataCachingProviderDecoratorTest.java +++ b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataCachingProviderDecoratorTest.java @@ -1,4 +1,4 @@ -package org.cryptomator.cloudaccess; +package org.cryptomator.cloudaccess.requestdecorator; import org.cryptomator.cloudaccess.api.CloudItemList; import org.cryptomator.cloudaccess.api.CloudItemMetadata; @@ -8,6 +8,7 @@ import org.cryptomator.cloudaccess.api.ProgressListener; import org.cryptomator.cloudaccess.api.exceptions.CloudProviderException; import org.cryptomator.cloudaccess.api.exceptions.NotFoundException; +import org.cryptomator.cloudaccess.requestdecorator.MetadataCachingProviderDecorator; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.Assertions; diff --git a/src/test/java/org/cryptomator/cloudaccess/MetadataRequestAggregatorProviderDecoratorTest.java b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestAggregatorProviderDecoratorTest.java similarity index 99% rename from src/test/java/org/cryptomator/cloudaccess/MetadataRequestAggregatorProviderDecoratorTest.java rename to src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestAggregatorProviderDecoratorTest.java index 092822d..0512958 100644 --- a/src/test/java/org/cryptomator/cloudaccess/MetadataRequestAggregatorProviderDecoratorTest.java +++ b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestAggregatorProviderDecoratorTest.java @@ -1,4 +1,4 @@ -package org.cryptomator.cloudaccess; +package org.cryptomator.cloudaccess.requestdecorator; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -10,6 +10,7 @@ import org.cryptomator.cloudaccess.api.ProgressListener; import org.cryptomator.cloudaccess.api.exceptions.CloudProviderException; import org.cryptomator.cloudaccess.api.exceptions.NotFoundException; +import org.cryptomator.cloudaccess.requestdecorator.MetadataCachingProviderDecorator; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.Assertions; diff --git a/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecoratorTest.java b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecoratorTest.java new file mode 100644 index 0000000..2f9eee5 --- /dev/null +++ b/src/test/java/org/cryptomator/cloudaccess/requestdecorator/MetadataRequestDeduplicationDecoratorTest.java @@ -0,0 +1,103 @@ +package org.cryptomator.cloudaccess.requestdecorator; + +import com.github.benmanes.caffeine.cache.AsyncCache; +import com.github.benmanes.caffeine.cache.Caffeine; +import org.cryptomator.cloudaccess.api.CloudItemList; +import org.cryptomator.cloudaccess.api.CloudItemMetadata; +import org.cryptomator.cloudaccess.api.CloudItemType; +import org.cryptomator.cloudaccess.api.CloudPath; +import org.cryptomator.cloudaccess.api.CloudProvider; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class MetadataRequestDeduplicationDecoratorTest { + + private final CloudPath file1 = CloudPath.of("/foo"); + private final CloudItemMetadata itemMetadata = new CloudItemMetadata(file1.getFileName().toString(), file1, CloudItemType.FILE); + private final CloudItemList itemList = new CloudItemList(List.of(itemMetadata)); + private final Optional pageToken = Optional.empty(); + + private CloudProvider cloudProvider; + private MetadataRequestDeduplicationDecorator decorator; + + private CompletableFuture futureItemMetadata1; + private CompletableFuture futureItemMetadata2; + private CompletableFuture futureItemList1; + private CompletableFuture futureItemList2; + + @BeforeEach + public void setup() { + cloudProvider = Mockito.mock(CloudProvider.class); + AsyncCache cachedItemMetadataRequests = Caffeine.newBuilder().expireAfterWrite(Duration.ofSeconds(0)).buildAsync(); + AsyncCache cachedItemListRequests = Caffeine.newBuilder().expireAfterWrite(Duration.ofSeconds(0)).buildAsync(); + decorator = new MetadataRequestDeduplicationDecorator(cloudProvider, cachedItemMetadataRequests, cachedItemListRequests); + + futureItemMetadata1 = new CompletableFuture<>(); + futureItemMetadata2 = new CompletableFuture<>(); + futureItemList1 = new CompletableFuture<>(); + futureItemList2 = new CompletableFuture<>(); + } + + @Test + @DisplayName("Same CompletionStage is returned as long as the future has not completed for itemMetadata") + public void testSameCompletionStageReturnedForItemMetadata() { + Mockito.doReturn(futureItemMetadata1, futureItemMetadata2).when(cloudProvider).itemMetadata(file1); + + var result1 = decorator.itemMetadata(file1); + var result2 = decorator.itemMetadata(file1); + + futureItemMetadata1.complete(itemMetadata); + + Assertions.assertSame(result1, result2); + Mockito.verify(cloudProvider, Mockito.atMostOnce()).itemMetadata(file1); + } + + @Test + @DisplayName("Different CompletionStage is returned after future has completed for itemMetadata") + public void testDifferentCompletionStageReturnedForItemMetadata() { + Mockito.doReturn(futureItemMetadata1, futureItemMetadata2).when(cloudProvider).itemMetadata(file1); + + var result1 = decorator.itemMetadata(file1); + futureItemMetadata1.complete(itemMetadata); + var result2 = decorator.itemMetadata(file1); + + Assertions.assertNotSame(result1, result2); + Mockito.verify(cloudProvider, Mockito.times(2)).itemMetadata(file1); + } + + @Test + @DisplayName("Same CompletionStage is returned as long as the future has not completed for list") + public void testSameCompletionStageReturnedForList() { + Mockito.doReturn(futureItemList1, futureItemList2).when(cloudProvider).list(file1, pageToken); + + var result1 = decorator.list(file1, pageToken); + var result2 = decorator.list(file1, pageToken); + + futureItemList1.complete(itemList); + + Assertions.assertSame(result1, result2); + Mockito.verify(cloudProvider, Mockito.atMostOnce()).list(file1, pageToken); + } + + @Test + @DisplayName("Different CompletionStage is returned after future has completed for list") + public void testDifferentCompletionStageReturnedForList() { + Mockito.doReturn(futureItemList1, futureItemList2).when(cloudProvider).list(file1, pageToken); + + var result1 = decorator.list(file1, pageToken); + futureItemList1.complete(itemList); + var result2 = decorator.list(file1, pageToken); + + Assertions.assertNotSame(result1, result2); + Mockito.verify(cloudProvider, Mockito.times(2)).list(file1, pageToken); + } + +} \ No newline at end of file