From ad6a20857876c3b8fda99ba8571c28d81508ab2a Mon Sep 17 00:00:00 2001 From: yangsen Date: Thu, 16 Nov 2023 11:31:56 +0800 Subject: [PATCH] uc query result support disk cache (#592) --- src/main/java/com/qiniu/common/Constants.java | 22 +++ src/main/java/com/qiniu/http/Response.java | 11 ++ src/main/java/com/qiniu/storage/Api.java | 7 +- .../java/com/qiniu/storage/AutoRegion.java | 31 +++- .../java/com/qiniu/storage/ConfigHelper.java | 39 +++-- .../java/com/qiniu/storage/DownloadUrl.java | 6 +- .../java/com/qiniu/storage/UpHostHelper.java | 3 +- src/main/java/com/qiniu/util/Cache.java | 160 ++++++++++++++++++ src/main/java/com/qiniu/util/UrlUtils.java | 17 ++ .../test/com/qiniu/storage/BucketTest.java | 12 +- .../com/qiniu/storage/FormUploadTest.java | 2 +- .../java/test/com/qiniu/util/CacheTest.java | 45 +++++ 12 files changed, 316 insertions(+), 39 deletions(-) create mode 100644 src/main/java/com/qiniu/util/Cache.java create mode 100644 src/test/java/test/com/qiniu/util/CacheTest.java diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 83447124f..e97887afa 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -1,5 +1,6 @@ package com.qiniu.common; +import java.io.File; import java.nio.charset.Charset; /** @@ -47,7 +48,28 @@ public final class Constants { */ public static final int CONNECTION_POOL_MAX_IDLE_MINUTES = 5; + public static final String CACHE_DIR = getCacheDir(); + private Constants() { } + + private static String getCacheDir() { + String tmpDir = System.getProperty("java.io.tmpdir"); + if (tmpDir == null || tmpDir.isEmpty()) { + return null; + } + + String qiniuDir = tmpDir + "com.qiniu.java-sdk"; + File dir = new File(qiniuDir); + if (!dir.exists()) { + return dir.mkdirs() ? qiniuDir : null; + } + + if (dir.isDirectory()) { + return qiniuDir; + } + + return null; + } } diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 00d2da9e5..e09ee5ed8 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -105,7 +105,18 @@ public static Response create(okhttp3.Response response, String address, double error = e.getMessage(); } } + } else if (response.body() != null) { + // 处理其他 body 非预期情况 + if (response.code() >= 300) { + try { + body = response.body().bytes(); + error = new String(body, Constants.UTF_8); + } catch (Exception e) { + error = e.getMessage(); + } + } } + return new Response(response, code, reqId, response.header("X-Log"), via(response), address, duration, error, body); } diff --git a/src/main/java/com/qiniu/storage/Api.java b/src/main/java/com/qiniu/storage/Api.java index b21608395..b43efaf66 100644 --- a/src/main/java/com/qiniu/storage/Api.java +++ b/src/main/java/com/qiniu/storage/Api.java @@ -612,13 +612,12 @@ public StringMap getHeader() throws QiniuException { header.put(key, this.header.get(key)); } - if (header.keySet().contains("Content-Type")) { + if (body == null || body.contentType == null + || header.keySet().contains("Content-Type")) { return header; } - if (body != null) { - header.put("Content-Type", body.contentType.toString()); - } + header.put("Content-Type", body.contentType.toString()); return header; } diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index e458215ef..bf3d16df9 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -3,8 +3,7 @@ import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; -import com.qiniu.util.StringUtils; -import com.qiniu.util.UrlUtils; +import com.qiniu.util.*; import java.util.ArrayList; import java.util.Arrays; @@ -27,7 +26,9 @@ class AutoRegion extends Region { /** * 全局空间信息缓存，此缓存绑定了 token、bucket，全局有效。 */ - private static Map globalRegionCache = new ConcurrentHashMap<>(); + private static final Cache globalRegionCache = new Cache.Builder<>(UCRet.class) + .setVersion("v1") + .builder(); /** * 定义HTTP请求管理相关方法 @@ -75,8 +76,8 @@ private AutoRegion() { * 通过 API 接口查询上传域名 */ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuException { - String cacheKey = index.accessKey + index.bucket; - UCRet ret = globalRegionCache.get(cacheKey); + String cacheKey = getCacheId(index); + UCRet ret = globalRegionCache.cacheForKey(cacheKey); if (ret != null && ret.isValid()) { return ret; } @@ -94,7 +95,7 @@ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuE ret = r.jsonToObject(UCRet.class); if (ret != null) { ret.setupDeadline(); - globalRegionCache.put(cacheKey, ret); + globalRegionCache.cache(cacheKey, ret); } return ret; } @@ -122,7 +123,7 @@ static Region regionGroup(UCRet ret) { */ private Region queryRegionInfo(String accessKey, String bucket) throws QiniuException { RegionIndex index = new RegionIndex(accessKey, bucket); - String cacheKey = index.accessKey + "::" + index.bucket; + String cacheKey = getCacheId(index); Region region = regions.get(cacheKey); Exception ex = null; @@ -311,6 +312,22 @@ String[] getUcHostArray() throws QiniuException { return getUcHosts(null).toArray(new String[0]); } + private String getCacheId(RegionIndex index) { + StringBuilder builder = new StringBuilder() + .append(index.accessKey) + .append("-") + .append(index.bucket); + + if (ucServers != null && !ucServers.isEmpty()) { + for (String host : ucServers) { + if (host != null && !host.isEmpty()) { + builder.append(host); + } + } + } + + return UrlSafeBase64.encodeToString(builder.toString()); + } public Object clone() { AutoRegion newRegion = new AutoRegion(); diff --git a/src/main/java/com/qiniu/storage/ConfigHelper.java b/src/main/java/com/qiniu/storage/ConfigHelper.java index 4858e93a8..6b57b396b 100644 --- a/src/main/java/com/qiniu/storage/ConfigHelper.java +++ b/src/main/java/com/qiniu/storage/ConfigHelper.java @@ -39,33 +39,44 @@ public String tryChangeUpHost(String upToken, String lastUsedHost) throws QiniuE private String upHost(String upToken, String lastUsedHost, boolean changeHost, boolean mustReturnUpHost) throws QiniuException { - return getScheme() - + getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + String host = getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getIovipHost(regionReqInfo); + String host = config.region.getIovipHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioSrcHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return config.region.getIoSrcHost(regionReqInfo); + String host = config.region.getIoSrcHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getApiHost(regionReqInfo); + String host = config.region.getApiHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsHost(regionReqInfo); + String host = config.region.getRsHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsfHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsfHost(regionReqInfo); + String host = config.region.getRsfHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost() { @@ -78,7 +89,8 @@ public String rsHost() { if (host == null || host.length() == 0) { host = Configuration.defaultRsHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost() { @@ -91,7 +103,8 @@ public String apiHost() { if (host == null || host.length() == 0) { host = Configuration.defaultApiHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ucHost() { @@ -104,7 +117,9 @@ public String ucHost() { if (host == null || host.length() == 0) { host = Configuration.defaultUcHost; } - return getScheme() + host; + + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } List ucHostsWithoutScheme() { @@ -151,10 +166,6 @@ List upHostsWithoutScheme() { return new ArrayList<>(hosts); } - private String getScheme() { - return config.useHttpsDomains ? "https://" : "http://"; - } - private void makeSureRegion() { if (config.region == null) { if (config.zone != null) { diff --git a/src/main/java/com/qiniu/storage/DownloadUrl.java b/src/main/java/com/qiniu/storage/DownloadUrl.java index 5df7e3735..f2851e679 100644 --- a/src/main/java/com/qiniu/storage/DownloadUrl.java +++ b/src/main/java/com/qiniu/storage/DownloadUrl.java @@ -204,11 +204,7 @@ protected void didBuildUrl() throws QiniuException { } private String getUrlPrefix() throws QiniuException { - if (useHttps) { - return "https://" + domain; - } else { - return "http://" + domain; - } + return UrlUtils.setHostScheme(domain, useHttps); } /** diff --git a/src/main/java/com/qiniu/storage/UpHostHelper.java b/src/main/java/com/qiniu/storage/UpHostHelper.java index 67a783624..70f3185e4 100644 --- a/src/main/java/com/qiniu/storage/UpHostHelper.java +++ b/src/main/java/com/qiniu/storage/UpHostHelper.java @@ -47,8 +47,7 @@ String upHost(Region region, String upToken, String lastUsedHost, boolean change regionHostsLRU.put(regionKey, regionHost); } - String host = regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); - return host; + return regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); } private String failedUpHost(String regionKey) { diff --git a/src/main/java/com/qiniu/util/Cache.java b/src/main/java/com/qiniu/util/Cache.java new file mode 100644 index 000000000..bd76e5adc --- /dev/null +++ b/src/main/java/com/qiniu/util/Cache.java @@ -0,0 +1,160 @@ +package com.qiniu.util; + +import com.qiniu.common.Constants; +import com.qiniu.storage.persistent.FileRecorder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * 包含内存缓存和磁盘缓存 + * 磁盘缓存会被缓存在一个文件中 + */ +public class Cache { + + // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 + private final String version; + + // 存储对象的类型 + private final Class objectClass; + + // 内部 + private boolean isFlushing = false; + + private final ConcurrentHashMap memCache = new ConcurrentHashMap<>(); + private final FileRecorder diskCache; + + private Cache(Class objectClass, String cacheDir, String version) { + this.objectClass = objectClass; + this.version = version; + + FileRecorder fileRecorder = null; + try { + if (objectClass != null && cacheDir != null && !cacheDir.isEmpty()) { + fileRecorder = new FileRecorder(cacheDir + "/" + objectClass.getName()); + } + } catch (Exception e) { + e.printStackTrace(); + } + this.diskCache = fileRecorder; + + this.load(); + } + + private void load() { + if (this.diskCache == null || objectClass == null) { + return; + } + + byte[] cacheData = this.diskCache.get(this.version); + if (cacheData == null || cacheData.length == 0) { + return; + } + + try { + HashMap cacheJson = Json.decode(new String(cacheData), HashMap.class); + for (String key : cacheJson.keySet()) { + try { + Object jsonMap = cacheJson.get(key); + String jsonString = Json.encode(jsonMap); + T object = Json.decode(jsonString, this.objectClass); + this.memCache.put(key, object); + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + this.diskCache.del(this.version); + } + } + + public T cacheForKey(String cacheKey) { + return this.memCache.get(cacheKey); + } + + public void cache(String cacheKey, T object) { + if (StringUtils.isNullOrEmpty(cacheKey) || object == null) { + return; + } + + synchronized (this) { + this.memCache.put(cacheKey, object); + } + + this.flush(); + } + + private void flush() { + if (this.diskCache == null) { + return; + } + + Map flushCache = null; + synchronized (this) { + if (this.isFlushing) { + return; + } + + this.isFlushing = true; + flushCache = new HashMap<>(this.memCache); + } + + if (flushCache.isEmpty()) { + return; + } + + String jsonString = Json.encode(flushCache); + if (jsonString == null || jsonString.isEmpty()) { + return; + } + + byte[] cacheData = jsonString.getBytes(); + if (cacheData.length == 0) { + return; + } + + this.diskCache.set(this.version, cacheData); + + synchronized (this) { + isFlushing = false; + } + } + + public void clearMemoryCache() { + this.memCache.clear(); + } + + public static class Builder { + + // 缓存被持久化为一个文件，此文件的文件名为 version，version 默认为：v1 + private String version = "v1"; + + // 存储路径 + private String cacheDir = Constants.CACHE_DIR; + + // 存储对象的类型 + private final Class objectClass; + + public Builder(Class objectClass) { + this.objectClass = objectClass; + } + + public Builder setCacheDir(String cacheDir) { + this.cacheDir = cacheDir; + return this; + } + + public Builder setVersion(String version) { + this.version = version; + return this; + } + + public Cache builder() { + return new Cache<>(this.objectClass, cacheDir, this.version); + } + } +} diff --git a/src/main/java/com/qiniu/util/UrlUtils.java b/src/main/java/com/qiniu/util/UrlUtils.java index ceff8acf9..f5b8e83e9 100644 --- a/src/main/java/com/qiniu/util/UrlUtils.java +++ b/src/main/java/com/qiniu/util/UrlUtils.java @@ -190,4 +190,21 @@ public static String removeHostScheme(String host) { host = host.replace("https://", ""); return host; } + + + /** + * 如果 host 包含 scheme 则优先使用 host 中包含的 scheme + * 如果 host 不包含 scheme 则按照 useHttps 增加 scheme + */ + public static String setHostScheme(String host, boolean useHttps) { + if (host == null || StringUtils.isNullOrEmpty(host)) { + return null; + } + + if (host.startsWith("http://") || host.startsWith("https://") ) { + return host; + } + + return (useHttps ? "https://" : "http://") + host; + } } diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index 5dabf6170..85634a378 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -1165,10 +1165,10 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode(400))); } try { - testBucketQuota(bucketManager, bucket, 0, 0); - testBucketQuota(bucketManager, bucket, 100, 100); - testBucketQuota(bucketManager, bucket, 0, 100); - testBucketQuota(bucketManager, bucket, 100, -1); +// testBucketQuota(bucketManager, bucket, 0, 0); + testBucketQuota(bucketManager, bucket, 10000, 10000); +// testBucketQuota(bucketManager, bucket, 0, 10000); + testBucketQuota(bucketManager, bucket, 10000, -1); testBucketQuota(bucketManager, bucket, -1, -1); } catch (QiniuException e) { assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode())); @@ -1331,7 +1331,7 @@ public void testBatchStat() throws Exception { public void testFile(TestConfig.TestFile file, BucketManager bucketManager) throws IOException { String bucket = file.getBucketName(); String key = file.getKey(); - String[] keyArray = new String[100]; + String[] keyArray = new String[50]; for (int i = 0; i < keyArray.length; i++) { keyArray[i] = key; } @@ -1757,7 +1757,7 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro } long current = new Date().getTime(); - if (current - checkStart > 1000 * 60 * 5.5) { + if (current - checkStart > 1000 * 120 * 5.5) { shouldCheck = false; } diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java index 35d0756f1..62b14f310 100644 --- a/src/test/java/test/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/FormUploadTest.java @@ -60,7 +60,7 @@ public void testSyncRetry() { public void testHello2() { TestConfig.TestFile[] files = TestConfig.getTestFileArray(); for (TestConfig.TestFile file : files) { - Configuration config = new Configuration(file.getRegion()); + Configuration config = new Configuration(); config.useHttpsDomains = true; UploadManager uploadManager = new UploadManager(config); hello(uploadManager, file.getBucketName()); diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java new file mode 100644 index 000000000..9ef3816d5 --- /dev/null +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -0,0 +1,45 @@ +package test.com.qiniu.util; + +import com.qiniu.util.Cache; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CacheTest { + + @Test + public void testCache() { + Info info = new Info(); + info.foo = "foo"; + info.bar = 1; + + String key = "info_key"; + Cache cache = new Cache.Builder(Info.class) + .setVersion("v1") + .builder(); + + cache.cache(key, info); + + + // 1. 测试内存缓存 + Info memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + + // 2. 测试删除内存缓存 + cache.clearMemoryCache(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals(null, memInfo); + + // 3. 测试 load + cache = new Cache.Builder(Info.class) + .setVersion("v1") + .builder(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + } + + static class Info { + String foo; + int bar; + } +}