Skip to content

Commit e6b04ec

Browse files
bjhargravebjhargrave
committed
repository: Add a Resource cache to FileSetRepository
The cache reduces the need to create new Resource objects, including SHA-256 computation, for unchanged files. Fixes #5367 Signed-off-by: BJ Hargrave <[email protected]>
1 parent e1eefb7 commit e6b04ec

File tree

4 files changed

+293
-54
lines changed

4 files changed

+293
-54
lines changed

Diff for: ‎biz.aQute.repository/src/aQute/bnd/repository/fileset/FileSetRepository.java

+6-54
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package aQute.bnd.repository.fileset;
22

3-
import static aQute.bnd.exceptions.FunctionWithException.asFunctionOrElse;
4-
53
import java.io.File;
64
import java.io.InputStream;
75
import java.net.URI;
@@ -10,10 +8,8 @@
108
import java.util.List;
119
import java.util.Map;
1210
import java.util.Objects;
13-
import java.util.Optional;
1411
import java.util.SortedSet;
1512

16-
import org.osgi.framework.namespace.IdentityNamespace;
1713
import org.osgi.resource.Capability;
1814
import org.osgi.resource.Requirement;
1915
import org.osgi.resource.Resource;
@@ -24,23 +20,17 @@
2420
import org.slf4j.LoggerFactory;
2521

2622
import aQute.bnd.exceptions.Exceptions;
27-
import aQute.bnd.osgi.Jar;
2823
import aQute.bnd.osgi.repository.BaseRepository;
2924
import aQute.bnd.osgi.repository.BridgeRepository;
3025
import aQute.bnd.osgi.repository.ResourcesRepository;
31-
import aQute.bnd.osgi.resource.CapReqBuilder;
32-
import aQute.bnd.osgi.resource.ResourceBuilder;
3326
import aQute.bnd.osgi.resource.ResourceUtils;
3427
import aQute.bnd.osgi.resource.ResourceUtils.ContentCapability;
3528
import aQute.bnd.service.Plugin;
3629
import aQute.bnd.service.Refreshable;
3730
import aQute.bnd.service.RepositoryPlugin;
3831
import aQute.bnd.util.repository.DownloadListenerPromise;
39-
import aQute.bnd.version.MavenVersion;
4032
import aQute.bnd.version.Version;
4133
import aQute.lib.strings.Strings;
42-
import aQute.maven.api.Revision;
43-
import aQute.maven.provider.POM;
4434
import aQute.service.reporter.Reporter;
4535

4636
public class FileSetRepository extends BaseRepository implements Plugin, RepositoryPlugin, Refreshable {
@@ -50,6 +40,7 @@ public class FileSetRepository extends BaseRepository implements Plugin, Reposit
5040
private volatile Deferred<BridgeRepository> repository;
5141
private Reporter reporter;
5242
private final PromiseFactory promiseFactory;
43+
private static final ResourceCache cache = new ResourceCache();
5344

5445
public FileSetRepository(String name, Collection<File> files) throws Exception {
5546
this.name = name;
@@ -90,53 +81,14 @@ private Promise<BridgeRepository> readFiles() {
9081
}
9182

9283
private Promise<Resource> parseFile(File file) {
93-
Promise<Resource> resource = promiseFactory.submit(() -> {
94-
if (!file.isFile()) {
95-
return null;
96-
}
97-
ResourceBuilder rb = new ResourceBuilder();
98-
try {
99-
boolean hasIdentity = rb.addFile(file, null);
100-
if (!hasIdentity) {
101-
try (Jar jar = new Jar(file)) {
102-
Optional<Revision> revision = jar.getPomXmlResources()
103-
.findFirst()
104-
.map(asFunctionOrElse(pomResource -> new POM(null, pomResource.openInputStream(), true),
105-
null))
106-
.map(POM::getRevision);
107-
108-
String name = jar.getModuleName();
109-
if (name == null) {
110-
name = revision.map(r -> r.program.toString())
111-
.orElse(null);
112-
if (name == null) {
113-
return null;
114-
}
115-
}
116-
117-
Version version = revision.map(r -> r.version.getOSGiVersion())
118-
.orElse(null);
119-
if (version == null) {
120-
version = new MavenVersion(jar.getModuleVersion()).getOSGiVersion();
121-
}
122-
123-
CapReqBuilder identity = new CapReqBuilder(IdentityNamespace.IDENTITY_NAMESPACE)
124-
.addAttribute(IdentityNamespace.IDENTITY_NAMESPACE, name)
125-
.addAttribute(IdentityNamespace.CAPABILITY_VERSION_ATTRIBUTE, version)
126-
.addAttribute(IdentityNamespace.CAPABILITY_TYPE_ATTRIBUTE, IdentityNamespace.TYPE_UNKNOWN);
127-
rb.addCapability(identity);
128-
}
129-
}
130-
} catch (Exception f) {
131-
return null;
132-
}
133-
logger.debug("{}: parsing {}", getName(), file);
134-
return rb.build();
84+
Promise<Resource> promise = promiseFactory.submit(() -> {
85+
Resource resource = cache.getResource(this, file);
86+
return resource;
13587
});
13688
if (logger.isDebugEnabled()) {
137-
resource.onFailure(failure -> logger.debug("{}: failed to parse {}", getName(), file, failure));
89+
promise.onFailure(failure -> logger.debug("{}: failed to parse {}", getName(), file, failure));
13890
}
139-
return resource;
91+
return promise;
14092
}
14193

14294
@Override

Diff for: ‎biz.aQute.repository/src/aQute/bnd/repository/fileset/ResourceCache.java

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package aQute.bnd.repository.fileset;
2+
3+
import static aQute.bnd.exceptions.FunctionWithException.asFunctionOrElse;
4+
5+
import java.io.File;
6+
import java.util.Map;
7+
import java.util.Optional;
8+
import java.util.concurrent.ConcurrentHashMap;
9+
import java.util.concurrent.TimeUnit;
10+
11+
import org.osgi.framework.namespace.IdentityNamespace;
12+
import org.osgi.resource.Resource;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
import aQute.bnd.osgi.Jar;
17+
import aQute.bnd.osgi.resource.CapReqBuilder;
18+
import aQute.bnd.osgi.resource.ResourceBuilder;
19+
import aQute.bnd.service.RepositoryPlugin;
20+
import aQute.bnd.version.MavenVersion;
21+
import aQute.bnd.version.Version;
22+
import aQute.maven.api.Revision;
23+
import aQute.maven.provider.POM;
24+
25+
final class ResourceCache {
26+
private final static Logger logger = LoggerFactory
27+
.getLogger(ResourceCache.class);
28+
private final static long EXPIRED_DURATION_NANOS = TimeUnit.NANOSECONDS.convert(30L,
29+
TimeUnit.MINUTES);
30+
private final Map<ResourceCacheKey, Resource> cache;
31+
private long time;
32+
33+
ResourceCache() {
34+
cache = new ConcurrentHashMap<>();
35+
time = System.nanoTime();
36+
}
37+
38+
Resource getResource(RepositoryPlugin repo, File file) {
39+
if (!file.isFile()) {
40+
return null;
41+
}
42+
// Make sure we don't grow infinitely
43+
final long now = System.nanoTime();
44+
if ((now - time) > EXPIRED_DURATION_NANOS) {
45+
cache.keySet()
46+
.removeIf(key -> (now - key.time) > EXPIRED_DURATION_NANOS);
47+
time = now;
48+
}
49+
ResourceCacheKey cacheKey = new ResourceCacheKey(file);
50+
Resource resource = cache.computeIfAbsent(cacheKey, key -> {
51+
ResourceBuilder rb = new ResourceBuilder();
52+
try {
53+
boolean hasIdentity = rb.addFile(file, null);
54+
if (!hasIdentity) {
55+
try (Jar jar = new Jar(file)) {
56+
Optional<Revision> revision = jar.getPomXmlResources()
57+
.findFirst()
58+
.map(asFunctionOrElse(pomResource -> new POM(null, pomResource.openInputStream(), true),
59+
null))
60+
.map(POM::getRevision);
61+
62+
String name = jar.getModuleName();
63+
if (name == null) {
64+
name = revision.map(r -> r.program.toString())
65+
.orElse(null);
66+
if (name == null) {
67+
return null;
68+
}
69+
}
70+
71+
Version version = revision.map(r -> r.version.getOSGiVersion())
72+
.orElse(null);
73+
if (version == null) {
74+
version = new MavenVersion(jar.getModuleVersion()).getOSGiVersion();
75+
}
76+
77+
CapReqBuilder identity = new CapReqBuilder(IdentityNamespace.IDENTITY_NAMESPACE)
78+
.addAttribute(IdentityNamespace.IDENTITY_NAMESPACE, name)
79+
.addAttribute(IdentityNamespace.CAPABILITY_VERSION_ATTRIBUTE, version)
80+
.addAttribute(IdentityNamespace.CAPABILITY_TYPE_ATTRIBUTE, IdentityNamespace.TYPE_UNKNOWN);
81+
rb.addCapability(identity);
82+
}
83+
}
84+
} catch (Exception f) {
85+
return null;
86+
}
87+
logger.debug("{}: parsing {}", repo.getName(), file);
88+
return rb.build();
89+
});
90+
91+
return resource;
92+
}
93+
}

Diff for: ‎biz.aQute.repository/src/aQute/bnd/repository/fileset/ResourceCacheKey.java

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package aQute.bnd.repository.fileset;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.nio.file.Files;
6+
import java.nio.file.Path;
7+
import java.nio.file.attribute.BasicFileAttributeView;
8+
import java.nio.file.attribute.BasicFileAttributes;
9+
import java.nio.file.attribute.FileTime;
10+
import java.util.Objects;
11+
import java.util.Optional;
12+
13+
import aQute.bnd.exceptions.Exceptions;
14+
15+
final class ResourceCacheKey {
16+
private final Path path;
17+
private final Object fileKey;
18+
private final FileTime lastModifiedTime;
19+
private final long size;
20+
final long time;
21+
22+
ResourceCacheKey(File file) {
23+
this(file.toPath());
24+
}
25+
26+
ResourceCacheKey(Path path) {
27+
path = path.toAbsolutePath();
28+
BasicFileAttributes attributes;
29+
try {
30+
attributes = Files.getFileAttributeView(path, BasicFileAttributeView.class)
31+
.readAttributes();
32+
} catch (IOException e) {
33+
throw Exceptions.duck(e);
34+
}
35+
if (!attributes.isRegularFile()) {
36+
throw new IllegalArgumentException("File must be a regular file: " + path);
37+
}
38+
this.path = path;
39+
this.fileKey = Optional.ofNullable(attributes.fileKey())
40+
.orElse(path); // Windows
41+
this.lastModifiedTime = attributes.lastModifiedTime();
42+
this.size = attributes.size();
43+
this.time = System.nanoTime();
44+
}
45+
46+
@Override
47+
public int hashCode() {
48+
return (Objects.hashCode(fileKey) * 31 + Objects.hashCode(lastModifiedTime)) * 31 + Long.hashCode(size);
49+
}
50+
51+
@Override
52+
public boolean equals(Object obj) {
53+
if (this == obj) {
54+
return true;
55+
}
56+
if (!(obj instanceof ResourceCacheKey)) {
57+
return false;
58+
}
59+
ResourceCacheKey other = (ResourceCacheKey) obj;
60+
return Objects.equals(fileKey, other.fileKey) && Objects.equals(lastModifiedTime, other.lastModifiedTime)
61+
&& (size == other.size);
62+
}
63+
64+
@Override
65+
public String toString() {
66+
return path.toString();
67+
}
68+
}

Diff for: ‎biz.aQute.repository/test/aQute/bnd/repository/fileset/RepositoryCacheKeyTest.java

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package aQute.bnd.repository.fileset;
2+
3+
import static org.assertj.core.api.Assertions.assertThatObject;
4+
5+
import java.nio.charset.StandardCharsets;
6+
import java.nio.file.Files;
7+
import java.nio.file.Path;
8+
import java.nio.file.attribute.BasicFileAttributeView;
9+
import java.nio.file.attribute.BasicFileAttributes;
10+
import java.nio.file.attribute.FileTime;
11+
import java.time.Instant;
12+
13+
import org.junit.jupiter.api.Test;
14+
import org.junit.jupiter.api.condition.DisabledOnOs;
15+
import org.junit.jupiter.api.condition.OS;
16+
17+
import aQute.bnd.test.jupiter.InjectTemporaryDirectory;
18+
import aQute.lib.io.IO;
19+
20+
class RepositoryCacheKeyTest {
21+
22+
@Test
23+
void unchanged(@InjectTemporaryDirectory
24+
Path tmp) throws Exception {
25+
Path subject = tmp.resolve("test");
26+
IO.store("line1", subject, StandardCharsets.UTF_8);
27+
ResourceCacheKey key1 = new ResourceCacheKey(subject);
28+
ResourceCacheKey key2 = new ResourceCacheKey(subject);
29+
assertThatObject(key1).isEqualTo(key2);
30+
assertThatObject(key1).hasSameHashCodeAs(key2);
31+
}
32+
33+
@Test
34+
void change_modified(@InjectTemporaryDirectory
35+
Path tmp) throws Exception {
36+
Path subject = tmp.resolve("test");
37+
IO.store("line1", subject, StandardCharsets.UTF_8);
38+
ResourceCacheKey key1 = new ResourceCacheKey(subject);
39+
BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class)
40+
.readAttributes();
41+
FileTime lastModifiedTime = attributes.lastModifiedTime();
42+
Instant plusSeconds = lastModifiedTime.toInstant()
43+
.plusSeconds(10L);
44+
Files.setLastModifiedTime(subject, FileTime.from(plusSeconds));
45+
ResourceCacheKey key2 = new ResourceCacheKey(subject);
46+
assertThatObject(key1).isNotEqualTo(key2);
47+
assertThatObject(key1).doesNotHaveSameHashCodeAs(key2);
48+
}
49+
50+
@Test
51+
void change_size(@InjectTemporaryDirectory
52+
Path tmp) throws Exception {
53+
Path subject = tmp.resolve("test");
54+
IO.store("line1", subject, StandardCharsets.UTF_8);
55+
ResourceCacheKey key1 = new ResourceCacheKey(subject);
56+
BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class)
57+
.readAttributes();
58+
FileTime lastModifiedTime = attributes.lastModifiedTime();
59+
IO.store("line100", subject, StandardCharsets.UTF_8);
60+
Files.setLastModifiedTime(subject, lastModifiedTime);
61+
ResourceCacheKey key2 = new ResourceCacheKey(subject);
62+
assertThatObject(key1).isNotEqualTo(key2);
63+
assertThatObject(key1).doesNotHaveSameHashCodeAs(key2);
64+
}
65+
66+
@DisabledOnOs(value = OS.WINDOWS, disabledReason = "Windows FS does not support fileKey")
67+
@Test
68+
void change_filekey(@InjectTemporaryDirectory
69+
Path tmp) throws Exception {
70+
Path subject = tmp.resolve("test");
71+
IO.store("line1", subject, StandardCharsets.UTF_8);
72+
ResourceCacheKey key1 = new ResourceCacheKey(subject);
73+
BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class)
74+
.readAttributes();
75+
assertThatObject(attributes.fileKey()).isNotNull();
76+
FileTime lastModifiedTime = attributes.lastModifiedTime();
77+
Path subject2 = tmp.resolve("test.tmp");
78+
IO.store("line2", subject2, StandardCharsets.UTF_8);
79+
Files.setLastModifiedTime(subject2, lastModifiedTime);
80+
IO.rename(subject2, subject);
81+
ResourceCacheKey key2 = new ResourceCacheKey(subject);
82+
attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class)
83+
.readAttributes();
84+
assertThatObject(attributes.fileKey()).isNotNull();
85+
assertThatObject(key1).as("key2 not equal")
86+
.isNotEqualTo(key2);
87+
assertThatObject(key1).as("key2 different hash")
88+
.doesNotHaveSameHashCodeAs(key2);
89+
}
90+
91+
@Test
92+
void change_file_modified(@InjectTemporaryDirectory
93+
Path tmp) throws Exception {
94+
Path subject = tmp.resolve("test");
95+
IO.store("line1", subject, StandardCharsets.UTF_8);
96+
ResourceCacheKey key1 = new ResourceCacheKey(subject);
97+
Path subject2 = tmp.resolve("test.tmp");
98+
IO.store("line2", subject2, StandardCharsets.UTF_8);
99+
BasicFileAttributes attributes = Files.getFileAttributeView(subject2, BasicFileAttributeView.class)
100+
.readAttributes();
101+
FileTime lastModifiedTime = attributes.lastModifiedTime();
102+
Instant plusSeconds = lastModifiedTime.toInstant()
103+
.plusSeconds(10L);
104+
Files.setLastModifiedTime(subject2, FileTime.from(plusSeconds));
105+
IO.rename(subject2, subject);
106+
ResourceCacheKey key2 = new ResourceCacheKey(subject);
107+
assertThatObject(key1).as("key2 not equal")
108+
.isNotEqualTo(key2);
109+
assertThatObject(key1).as("key2 different hash")
110+
.doesNotHaveSameHashCodeAs(key2);
111+
}
112+
113+
@Test
114+
void different_files(@InjectTemporaryDirectory
115+
Path tmp) throws Exception {
116+
Path subject1 = tmp.resolve("test1");
117+
IO.store("line1", subject1, StandardCharsets.UTF_8);
118+
ResourceCacheKey key1 = new ResourceCacheKey(subject1);
119+
Path subject2 = tmp.resolve("test2");
120+
IO.copy(subject1, subject2);
121+
ResourceCacheKey key2 = new ResourceCacheKey(subject2);
122+
assertThatObject(key1).isNotEqualTo(key2);
123+
assertThatObject(key1).doesNotHaveSameHashCodeAs(key2);
124+
}
125+
126+
}

0 commit comments

Comments
 (0)