From a5b7adb32a099bee3ade2b7a4bcad977a72143c6 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 17 Jul 2023 15:30:56 +0800 Subject: [PATCH 01/19] version to 7.14.0 --- CHANGELOG.md | 9 +++++++-- README.md | 4 ++-- src/main/java/com/qiniu/common/Constants.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88fc00ac8..85e69e3ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ # Changelog +## 7.14.0(2023-07-18) +* Region 移除亚太首尔区域 +* BucketManager 的部分 Bucket 级别接口支持主备重试 + ## 7.13.1(2023-05-25) -* GetBucketQuota -* ChangeApiHost +* 优化 BucketManager 的 getBucketQuota 方法 +* 废弃 pu.qbox.me 域名 +* 废弃 BucketManager 的 listFilesV2 方法 * PlayExpire ## 7.13.0(2023-03-30) diff --git a/README.md b/README.md index 0950f6efb..d30d2398c 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,12 @@ com.qiniu qiniu-java-sdk - [7.13.0, 7.13.99] + [7.14.0, 7.14.99] ``` 或者 Gradle: ```groovy -implementation 'com.qiniu:qiniu-java-sdk:7.13.+' +implementation 'com.qiniu:qiniu-java-sdk:7.14.+' ``` ## 运行环境 diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index a1cd6655d..83447124f 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -9,7 +9,7 @@ public final class Constants { /** * 版本号 */ - public static final String VERSION = "7.13.1"; + public static final String VERSION = "7.14.0"; /** * 块大小,不能改变 */ From e47717965fab7603d11853f8d289dd636772e85e Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 11:35:39 +0800 Subject: [PATCH 02/19] Region support create by region id --- .../storage/ApiInterceptorRetryHosts.java | 4 ++ .../storage/ApiInterceptorRetrySimple.java | 4 ++ src/main/java/com/qiniu/storage/Region.java | 49 +++++++++++-- src/test/java/test/com/qiniu/TestConfig.java | 4 +- .../test/com/qiniu/storage/RegionTest.java | 70 +++++++++++++++++++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 src/test/java/test/com/qiniu/storage/RegionTest.java diff --git a/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java b/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java index 92878e65e..343ac2b5f 100644 --- a/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java +++ b/src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java @@ -84,6 +84,10 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc request.setHost(newHost); } + if (response != null && response.getResponse() != null) { + response.getResponse().close(); + } + int interval = retryInterval.interval(); if (interval <= 0) { continue; diff --git a/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java b/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java index befd92c67..64768bb4d 100644 --- a/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java +++ b/src/main/java/com/qiniu/storage/ApiInterceptorRetrySimple.java @@ -50,6 +50,10 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc } request = cloneRequest; + if (response != null && response.getResponse() != null) { + response.getResponse().close(); + } + int interval = retryInterval.interval(); if (interval <= 0) { continue; diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index e9d75becb..35fcb3f86 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -57,9 +57,11 @@ public class Region implements Cloneable { /** * 华东机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z0 * * @return 区域信息 */ + @Deprecated public static Region region0() { return new Builder(). region("z0"). @@ -74,9 +76,11 @@ public static Region region0() { /** * 华东机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z0 * * @return 区域信息 */ + @Deprecated public static Region huadong() { return region0(); } @@ -84,9 +88,11 @@ public static Region huadong() { /** * 华东浙江 2 机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:cn-east-2 * * @return 区域信息 */ + @Deprecated public static Region regionCnEast2() { return new Builder(). region("cn-east-2"). @@ -101,9 +107,11 @@ public static Region regionCnEast2() { /** * 华东浙江 2 机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:cn-east-2 * * @return 区域信息 */ + @Deprecated public static Region huadongZheJiang2() { return regionCnEast2(); } @@ -137,9 +145,11 @@ public static Region qvmHuadong() { /** * 华北机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z1 * * @return 区域信息 */ + @Deprecated public static Region region1() { return new Builder(). region("z1"). @@ -154,9 +164,11 @@ public static Region region1() { /** * 华北机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z1 * * @return 区域信息 */ + @Deprecated public static Region huabei() { return region1(); } @@ -189,9 +201,11 @@ public static Region qvmHuabei() { /** * 华南机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z2 * * @return 区域信息 */ + @Deprecated public static Region region2() { return new Builder(). region("z2"). @@ -206,18 +220,22 @@ public static Region region2() { /** * 华南机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:z2 * * @return 区域信息 */ + @Deprecated public static Region huanan() { return region2(); } /** * 北美机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:na0 * * @return 区域信息 */ + @Deprecated public static Region regionNa0() { return new Builder(). region("na0"). @@ -232,21 +250,25 @@ public static Region regionNa0() { /** * 北美机房相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:na0 * * @return 区域信息 */ + @Deprecated public static Region beimei() { return regionNa0(); } /** * 新加坡相关域名 + * 废弃,使用 {@link Region#createWithRegionId} 替换,regionId:as0 * * @return 区域信息 */ + @Deprecated public static Region regionAs0() { return new Builder(). - region("na0"). + region("as0"). srcUpHost("up-as0.qiniup.com"). accUpHost("upload-as0.qiniup.com"). iovipHost("iovip-as0.qbox.me"). @@ -256,6 +278,25 @@ public static Region regionAs0() { build(); } + /** + * Region 构造方法 + * regionId 参考链接: + * + * @param regionId 根据区域 ID 创建 Region + * @return Region 实例 + */ + public static Region createWithRegionId(String regionId) { + return new Builder() + .region(regionId) + .srcUpHost("up-" + regionId + ".qiniup.com") + .accUpHost("upload-" + regionId + ".qiniup.com") + .iovipHost("iovip-" + regionId + ".qiniuio.com") + .rsHost("rs-" + regionId + ".qiniuapi.com") + .rsfHost("rsf-" + regionId + ".qiniuapi.com") + .apiHost("api-" + regionId + ".qiniuapi.com") + .build(); + } + /** * 新加坡机房相关域名 * @@ -434,10 +475,10 @@ public Region autoRegion(String... ucServers) { /** * 自动选择,其它参数设置无效 * - * @param retryMax 单个域名最大重试次数 - * @param retryInterval 重试间隔,单位:毫秒 + * @param retryMax 单个域名最大重试次数 + * @param retryInterval 重试间隔,单位:毫秒 * @param hostFreezeDuration 冻结时间,单位:毫秒;域名请求失败会被冻结,冻结后域名在冻结时间内不会被使用 - * @param ucServers uc host + * @param ucServers uc host * @return 区域信息 **/ public Region autoRegion(int retryMax, int retryInterval, int hostFreezeDuration, String... ucServers) { diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index e7162328b..788268613 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -98,7 +98,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp na0.testDomainTimeStamp = testDomain_na0_timeStamp; na0.testUrlTimeStamp = "http://" + testDomain_na0_timeStamp + "/" + fileSaveKey; na0.regionId = "na0"; - na0.region = Region.regionNa0(); + na0.region = Region.createWithRegionId("na0"); TestFile z0 = new TestFile(); z0.key = fileSaveKey; @@ -109,7 +109,7 @@ public static TestFile[] getTestFileArray(String fileSaveKey, String fileMimeTyp z0.testDomainTimeStamp = testDomain_z0_timeStamp; z0.testUrlTimeStamp = "http://" + testDomain_z0_timeStamp + "/" + fileSaveKey; z0.regionId = "z0"; - z0.region = Region.region0(); + z0.region = Region.createWithRegionId("z0"); TestFile z0_auto = new TestFile(); z0_auto.key = fileSaveKey; diff --git a/src/test/java/test/com/qiniu/storage/RegionTest.java b/src/test/java/test/com/qiniu/storage/RegionTest.java new file mode 100644 index 000000000..9a63b3ef4 --- /dev/null +++ b/src/test/java/test/com/qiniu/storage/RegionTest.java @@ -0,0 +1,70 @@ +package test.com.qiniu.storage; + +import com.qiniu.storage.Region; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class RegionTest { + + @Test + @Tag("UnitTest") + public void testCreateWithRegionId() throws Exception { + Class regionReqInfoClass = Class.forName("com.qiniu.storage.RegionReqInfo"); + Constructor regionReqInfoConstructor = regionReqInfoClass.getDeclaredConstructor(String.class, String.class); + regionReqInfoConstructor.setAccessible(true); + + Object info = regionReqInfoConstructor.newInstance("a", "b"); + Region na0 = Region.createWithRegionId("na0"); + + // region id + Method getRegion = Region.class.getDeclaredMethod("getRegion", regionReqInfoClass); + getRegion.setAccessible(true); + String regionId = (String) getRegion.invoke(na0, info); + assertEquals("na0", regionId); + + // up + // up src + Method getSrcUpHost = Region.class.getDeclaredMethod("getSrcUpHost", regionReqInfoClass); + getSrcUpHost.setAccessible(true); + List srcHosts = (List) getSrcUpHost.invoke(na0, info); + assertTrue(srcHosts.contains("up-na0.qiniup.com"), "check up src error"); + + // up acc + Method getAccUpHost = Region.class.getDeclaredMethod("getAccUpHost", regionReqInfoClass); + getAccUpHost.setAccessible(true); + List accHosts = (List) getAccUpHost.invoke(na0, info); + assertTrue(accHosts.contains("upload-na0.qiniup.com"), "check up acc error"); + + // io + Method getIoHost = Region.class.getDeclaredMethod("getIovipHost", regionReqInfoClass); + getIoHost.setAccessible(true); + String ioHost = (String) getIoHost.invoke(na0, info); + assertTrue(ioHost.equals("iovip-na0.qiniuio.com"), "check io error"); + + // rs + Method getRsHost = Region.class.getDeclaredMethod("getRsHost", regionReqInfoClass); + getRsHost.setAccessible(true); + String rsHost = (String) getRsHost.invoke(na0, info); + assertTrue(rsHost.equals("rs-na0.qiniuapi.com"), "check rs error"); + + // rsf + Method getRsfHost = Region.class.getDeclaredMethod("getRsfHost", regionReqInfoClass); + getRsfHost.setAccessible(true); + String rsfHost = (String) getRsfHost.invoke(na0, info); + assertTrue(rsfHost.equals("rsf-na0.qiniuapi.com"), "check rsf error"); + + // api + Method getApiHost = Region.class.getDeclaredMethod("getApiHost", regionReqInfoClass); + getApiHost.setAccessible(true); + String apiHost = (String) getApiHost.invoke(na0, info); + assertTrue(apiHost.equals("api-na0.qiniuapi.com"), "check api error"); + } + +} From ba3073bf1a93ec101e83b1227cd9d20aca5120a4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 15:20:47 +0800 Subject: [PATCH 03/19] test case: exception don't print --- src/main/java/com/qiniu/http/Response.java | 2 +- src/main/java/com/qiniu/storage/BucketManager.java | 1 + src/test/java/test/com/qiniu/sms/SmsTest.java | 1 - .../java/test/com/qiniu/storage/ApiUploadV2Test.java | 5 ++--- src/test/java/test/com/qiniu/storage/BucketTest.java | 2 +- src/test/java/test/com/qiniu/storage/BucketTest2.java | 11 ++++------- .../test/com/qiniu/storage/FixBlockUploaderTest.java | 3 +-- .../java/test/com/qiniu/storage/FormUploadTest.java | 2 -- .../java/test/com/qiniu/storage/RecordUploadTest.java | 3 +-- 9 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 9461f9fec..00d2da9e5 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -248,7 +248,7 @@ public synchronized InputStream bodyStream() throws QiniuException { } public synchronized void close() { - if (this.response != null) { + if (this.response != null && this.response.body() != null) { this.response.close(); } } diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index d3100be0d..91eeba1fc 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -238,6 +238,7 @@ public FileListing listFiles(String bucket, String prefix, String marker, int li throws QiniuException { Response response = listV1(bucket, prefix, marker, limit, delimiter); if (!response.isOK()) { + response.close(); throw new QiniuException(response); } FileListing fileListing = response.jsonToObject(FileListing.class); diff --git a/src/test/java/test/com/qiniu/sms/SmsTest.java b/src/test/java/test/com/qiniu/sms/SmsTest.java index 9b8616be8..a41054c1e 100644 --- a/src/test/java/test/com/qiniu/sms/SmsTest.java +++ b/src/test/java/test/com/qiniu/sms/SmsTest.java @@ -102,7 +102,6 @@ public void sendFulltextMessage() { Response response = smsManager.sendFulltextMessage(mobiles, "【七牛云】尊敬的用户你好,您的验证码是 38232"); assertNotNull(response); } catch (QiniuException e) { - e.printStackTrace(); assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode(401))); } } diff --git a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java index e315171b7..dc336084b 100644 --- a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java +++ b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java @@ -323,8 +323,7 @@ public void testAbortUpload() { assertTrue(initUploadResponse.isOK(), initUploadResponse.getResponse() + ""); assertNotNull(initUploadResponse.getUploadId(), initUploadResponse.getUploadId() + ""); assertNotNull(initUploadResponse.getExpireAt(), initUploadResponse.getExpireAt() + ""); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } // 2. 上传文件数据 @@ -393,7 +392,7 @@ public void testAbortUpload() { // 列举结束 break; } catch (QiniuException e) { - e.printStackTrace(); +// e.printStackTrace(); assertTrue(e.response.statusCode == 612, e.response + ""); break; } diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index 5f58384e5..f6d50ad10 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -220,7 +220,7 @@ public void testListIterator() throws Exception { testFileWithHandler(new TestFileHandler() { @Override public void testFile(TestConfig.TestFile file, BucketManager bucketManager) throws IOException { - BucketManager.FileListIterator it = bucketManager.createFileListIterator(file.getBucketName(), "", 20, + BucketManager.FileListIterator it = bucketManager.createFileListIterator(file.getBucketName(), "", 500, null); assertTrue(it.hasNext()); FileInfo[] items0 = it.next(); diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index bc01a89a4..bfa247cac 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -139,7 +139,7 @@ public void testListUseDelimiter() { public void testListIterator() { String[] buckets = new String[]{TestConfig.testBucket_z0}; for (String bucket : buckets) { - BucketManager.FileListIterator it = bucketManager.createFileListIterator(bucket, "", 300, null); + BucketManager.FileListIterator it = bucketManager.createFileListIterator(bucket, "", 500, null); assertTrue(it.hasNext()); FileInfo[] items0 = it.next(); @@ -230,8 +230,7 @@ public void testStat() { try { bucketManager.deleteBucketLifecycleRule(TestConfig.testBucket_z0, ruleName); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } try { @@ -265,8 +264,7 @@ public void testStat() { } finally { try { bucketManager.deleteBucketLifecycleRule(TestConfig.testBucket_z0, ruleName); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } } @@ -1470,8 +1468,7 @@ public void testIndexPage() throws QiniuException { try { bucketManager.setIndexPage(TestConfig.dummyBucket, IndexPageType.HAS); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { } } diff --git a/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java b/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java index f3e7b2372..b02e4f1a9 100644 --- a/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java +++ b/src/test/java/test/com/qiniu/storage/FixBlockUploaderTest.java @@ -232,8 +232,7 @@ public void testKey2() throws IOException { try { Response res = up.upload(f, token, null); fail("key not match, should failed"); - } catch (QiniuException e) { - e.printStackTrace(); + } catch (QiniuException ignore) { // TODO } } diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java index 1948c839a..35d0756f1 100644 --- a/src/test/java/test/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/FormUploadTest.java @@ -146,8 +146,6 @@ public void testInvalidToken() { if (e.code() != -1) { assertEquals(401, e.code()); assertNotNull(e.response.reqId); - } else { - e.printStackTrace(); } } } diff --git a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java index 6c0be4f9b..0cd9314ee 100644 --- a/src/test/java/test/com/qiniu/storage/RecordUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/RecordUploadTest.java @@ -87,9 +87,8 @@ public void run() { System.out.println("UP: " + i + ", enter run"); response = up.up(region); System.out.println("UP: " + i + ", left run"); - } catch (Exception e) { + } catch (Exception ignore) { System.out.println("UP: " + i + ", exception run"); - e.printStackTrace(); } complete.isComplete.set(true); } From 915e96c3f6b70e27058f88c81c8a61db3ffbec05 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 27 Oct 2023 18:29:29 +0800 Subject: [PATCH 04/19] change test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 1 - src/test/java/test/com/qiniu/storage/BucketTest2.java | 1 - src/test/java/test/com/qiniu/storage/DownloadUrlTest.java | 3 +++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index f6d50ad10..5dabf6170 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -525,7 +525,6 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro fRet = bucketManager.fetch(resUrl, file.getBucketName()); assertEquals(resHash, fRet.hash); } catch (QiniuException e) { - e.printStackTrace(); // use e.response.toString() may get NullPointException // when java.net.SocketTimeoutException: timeout fail(e.getMessage()); diff --git a/src/test/java/test/com/qiniu/storage/BucketTest2.java b/src/test/java/test/com/qiniu/storage/BucketTest2.java index bfa247cac..a1498b16a 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest2.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest2.java @@ -499,7 +499,6 @@ public void testFetch() { fRet = bucketManager.fetch(resUrl, bucket); assertEquals(resHash, fRet.hash); } catch (QiniuException e) { - e.printStackTrace(); // use e.response.toString() may get NullPointException // when java.net.SocketTimeoutException: timeout fail(e.getMessage()); diff --git a/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java index 567f7e7a4..5afac57e9 100644 --- a/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java +++ b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java @@ -100,6 +100,7 @@ public void testUrlWithDeadline() { System.out.println("create url:" + url); Client client = new Client(); Response response = client.get(url); + response.close(); assertTrue(response.isOK(), response.toString()); try { @@ -120,6 +121,7 @@ private void testNoAuthority(String url) { try { Client client = new Client(); Response response = client.get(url); + response.close(); assertFalse(response.isOK(), url); } catch (QiniuException e) { assertNotNull(e.response, "except no authority:" + url + "\n but no response:" + e); @@ -131,6 +133,7 @@ private void testHasAuthority(String url) { try { Client client = new Client(); Response response = client.get(url); + response.close(); assertTrue(response.isOK(), url); } catch (QiniuException e) { fail("except has authority:" + url + "\n response:" + e.response); From f4a48e2dec805a7f14d8fd73e9f2c4bb436b5182 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 30 Oct 2023 10:08:37 +0800 Subject: [PATCH 05/19] change test case --- src/test/java/test/com/qiniu/CdnTest.java | 4 +++- src/test/java/test/com/qiniu/streaming/StreamingTest.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/CdnTest.java b/src/test/java/test/com/qiniu/CdnTest.java index 92852bcbb..4007965d2 100644 --- a/src/test/java/test/com/qiniu/CdnTest.java +++ b/src/test/java/test/com/qiniu/CdnTest.java @@ -176,7 +176,9 @@ public void testCreateTimestampAntiLeechUrlSimple() { String testUrl_z0_timeStamp_outdate = "http://javasdk-timestamp.peterpy.cn/do_not_delete/1.png?sign=14f48f829b78d5c9a34eb77e9a13f1b6&t=5890f13a"; try { URL url = new URL(TestConfig.testUrl_z0_timeStamp); - assertEquals(403, getResponse(url.toString()).statusCode, msg); + Response response = getResponse(url.toString()); + response.close(); + assertEquals(403, response.statusCode, msg); String signedUrl1 = CdnManager.createTimestampAntiLeechUrl(host, fileName, queryStringMap, encryptKey1, deadline1); String signedUrl2 = CdnManager.createTimestampAntiLeechUrl(url, encryptKey2, deadline2); diff --git a/src/test/java/test/com/qiniu/streaming/StreamingTest.java b/src/test/java/test/com/qiniu/streaming/StreamingTest.java index 9b515c8af..4aae60eb0 100644 --- a/src/test/java/test/com/qiniu/streaming/StreamingTest.java +++ b/src/test/java/test/com/qiniu/streaming/StreamingTest.java @@ -56,8 +56,7 @@ public void testStreamOperation() throws QiniuException { try { // 确保流存在 // manager.create(stream); - } catch (Exception e) { - e.printStackTrace(); + } catch (Exception ignore) { } StreamAttribute attr = manager.attribute(stream); assertEquals(0, attr.disabledTill); From e95a91fca02285f8db2d61a79c5f532e88ecd387 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 30 Oct 2023 11:04:55 +0800 Subject: [PATCH 06/19] change test case --- src/test/java/test/com/qiniu/CdnTest.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/CdnTest.java b/src/test/java/test/com/qiniu/CdnTest.java index 4007965d2..b5dbc6f14 100644 --- a/src/test/java/test/com/qiniu/CdnTest.java +++ b/src/test/java/test/com/qiniu/CdnTest.java @@ -186,9 +186,16 @@ public void testCreateTimestampAntiLeechUrlSimple() { System.out.println(signedUrl1); System.out.println(signedUrl2); System.out.println(signedUrl3); - assertEquals(200, getResponse(signedUrl1).statusCode, msg); - assertEquals(200, getResponse(signedUrl2).statusCode, msg); - assertEquals(403, getResponse(signedUrl3).statusCode, msg); + + response = getResponse(signedUrl1); + response.close(); + assertEquals(200, response.statusCode, msg); + response = getResponse(signedUrl2); + response.close(); + assertEquals(200, response.statusCode, msg); + response = getResponse(signedUrl3); + response.close(); + assertEquals(403, response.statusCode, msg); assertEquals(testUrl_z0_timeStamp_outdate, signedUrl3); } catch (Exception ex) { ex.printStackTrace(); From 18ba83ba7b110300baa7ce0d3d1e8fdbe7650074 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 2 Nov 2023 14:29:09 +0800 Subject: [PATCH 07/19] uc query cache support disk --- src/main/java/com/qiniu/common/Constants.java | 22 +++ .../java/com/qiniu/storage/AutoRegion.java | 31 ++- src/main/java/com/qiniu/util/Cache.java | 179 ++++++++++++++++++ .../com/qiniu/storage/FormUploadTest.java | 2 +- .../java/test/com/qiniu/util/CacheTest.java | 50 +++++ 5 files changed, 276 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/qiniu/util/Cache.java create mode 100644 src/test/java/test/com/qiniu/util/CacheTest.java diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 83447124f..e97887afa 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -1,5 +1,6 @@ package com.qiniu.common; +import java.io.File; import java.nio.charset.Charset; /** @@ -47,7 +48,28 @@ public final class Constants { */ public static final int CONNECTION_POOL_MAX_IDLE_MINUTES = 5; + public static final String CACHE_DIR = getCacheDir(); + private Constants() { } + + private static String getCacheDir() { + String tmpDir = System.getProperty("java.io.tmpdir"); + if (tmpDir == null || tmpDir.isEmpty()) { + return null; + } + + String qiniuDir = tmpDir + "com.qiniu.java-sdk"; + File dir = new File(qiniuDir); + if (!dir.exists()) { + return dir.mkdirs() ? qiniuDir : null; + } + + if (dir.isDirectory()) { + return qiniuDir; + } + + return null; + } } diff --git a/src/main/java/com/qiniu/storage/AutoRegion.java b/src/main/java/com/qiniu/storage/AutoRegion.java index e458215ef..bf3d16df9 100644 --- a/src/main/java/com/qiniu/storage/AutoRegion.java +++ b/src/main/java/com/qiniu/storage/AutoRegion.java @@ -3,8 +3,7 @@ import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.Response; -import com.qiniu.util.StringUtils; -import com.qiniu.util.UrlUtils; +import com.qiniu.util.*; import java.util.ArrayList; import java.util.Arrays; @@ -27,7 +26,9 @@ class AutoRegion extends Region { /** * 全局空间信息缓存,此缓存绑定了 token、bucket,全局有效。 */ - private static Map globalRegionCache = new ConcurrentHashMap<>(); + private static final Cache globalRegionCache = new Cache.Builder<>(UCRet.class) + .setVersion("v1") + .builder(); /** * 定义HTTP请求管理相关方法 @@ -75,8 +76,8 @@ private AutoRegion() { * 通过 API 接口查询上传域名 */ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuException { - String cacheKey = index.accessKey + index.bucket; - UCRet ret = globalRegionCache.get(cacheKey); + String cacheKey = getCacheId(index); + UCRet ret = globalRegionCache.cacheForKey(cacheKey); if (ret != null && ret.isValid()) { return ret; } @@ -94,7 +95,7 @@ private UCRet queryRegionInfoFromServerIfNeeded(RegionIndex index) throws QiniuE ret = r.jsonToObject(UCRet.class); if (ret != null) { ret.setupDeadline(); - globalRegionCache.put(cacheKey, ret); + globalRegionCache.cache(cacheKey, ret); } return ret; } @@ -122,7 +123,7 @@ static Region regionGroup(UCRet ret) { */ private Region queryRegionInfo(String accessKey, String bucket) throws QiniuException { RegionIndex index = new RegionIndex(accessKey, bucket); - String cacheKey = index.accessKey + "::" + index.bucket; + String cacheKey = getCacheId(index); Region region = regions.get(cacheKey); Exception ex = null; @@ -311,6 +312,22 @@ String[] getUcHostArray() throws QiniuException { return getUcHosts(null).toArray(new String[0]); } + private String getCacheId(RegionIndex index) { + StringBuilder builder = new StringBuilder() + .append(index.accessKey) + .append("-") + .append(index.bucket); + + if (ucServers != null && !ucServers.isEmpty()) { + for (String host : ucServers) { + if (host != null && !host.isEmpty()) { + builder.append(host); + } + } + } + + return UrlSafeBase64.encodeToString(builder.toString()); + } public Object clone() { AutoRegion newRegion = new AutoRegion(); diff --git a/src/main/java/com/qiniu/util/Cache.java b/src/main/java/com/qiniu/util/Cache.java new file mode 100644 index 000000000..0cc297487 --- /dev/null +++ b/src/main/java/com/qiniu/util/Cache.java @@ -0,0 +1,179 @@ +package com.qiniu.util; + +import com.qiniu.common.Constants; +import com.qiniu.storage.persistent.FileRecorder; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * 包含内存缓存和磁盘缓存 + * 磁盘缓存会被缓存在一个文件中 + */ +public class Cache { + + // 当 cache 修改数量到达这个值时,就会 flush,默认是 1 + private final int flushCount; + + // 缓存被持久化为一个文件,此文件的文件名为 version,version 默认为:v1 + private final String version; + + // 存储对象的类型 + private final Class objectClass; + + // 内部 + private boolean isFlushing = false; + private int needFlushCount = 0; + + private final ConcurrentHashMap memCache = new ConcurrentHashMap<>(); + private final FileRecorder diskCache; + + private Cache(Class objectClass, String cacheDir, int flushCount, String version) { + this.objectClass = objectClass; + this.flushCount = flushCount; + this.version = version; + + FileRecorder fileRecorder = null; + try { + if (objectClass != null && cacheDir != null && !cacheDir.isEmpty()) { + fileRecorder = new FileRecorder(cacheDir + "/" + objectClass.getName()); + } + } catch (Exception e) { + e.printStackTrace(); + } + this.diskCache = fileRecorder; + + this.load(); + } + + private void load() { + if (this.diskCache == null || objectClass == null) { + return; + } + + byte[] cacheData = this.diskCache.get(this.version); + if (cacheData == null || cacheData.length == 0) { + return; + } + + try { + HashMap cacheJson = Json.decode(new String(cacheData), HashMap.class); + for (String key : cacheJson.keySet()) { + try { + Object jsonMap = cacheJson.get(key); + String jsonString = Json.encode(jsonMap); + T object = Json.decode(jsonString, this.objectClass); + this.memCache.put(key, object); + } catch (Exception e) { + e.printStackTrace(); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + this.diskCache.del(this.version); + } + } + + public T cacheForKey(String cacheKey) { + return this.memCache.get(cacheKey); + } + + public void cache(String cacheKey, T object) { + if (StringUtils.isNullOrEmpty(cacheKey) || object == null) { + return; + } + + synchronized (this) { + this.needFlushCount++; + this.memCache.put(cacheKey, object); + + if (this.needFlushCount < this.flushCount) { + return; + } + + this.needFlushCount = 0; + } + + this.flush(); + } + + private void flush() { + if (this.diskCache == null) { + return; + } + + Map flushCache = null; + synchronized (this) { + if (this.isFlushing) { + return; + } + + this.isFlushing = true; + flushCache = new HashMap<>(this.memCache); + } + + if (flushCache.isEmpty()) { + return; + } + + String jsonString = Json.encode(flushCache); + if (jsonString == null || jsonString.isEmpty()) { + return; + } + + byte[] cacheData = jsonString.getBytes(); + if (cacheData.length == 0) { + return; + } + + this.diskCache.set(this.version, cacheData); + + synchronized (this) { + isFlushing = false; + } + } + + public void clearMemoryCache() { + this.memCache.clear(); + } + + public static class Builder { + // 当 cache 修改数量到达这个值时,就会 flush,默认是 1 + private int flushCount = 1; + + // 缓存被持久化为一个文件,此文件的文件名为 version,version 默认为:v1 + private String version = "v1"; + + // 存储路径 + private String cacheDir = Constants.CACHE_DIR; + + // 存储对象的类型 + private final Class objectClass; + + public Builder(Class objectClass) { + this.objectClass = objectClass; + } + + public Builder setFlushCount(int flushCount) { + this.flushCount = flushCount; + return this; + } + + public Builder setCacheDir(String cacheDir) { + this.cacheDir = cacheDir; + return this; + } + + public Builder setVersion(String version) { + this.version = version; + return this; + } + + public Cache builder() { + return new Cache<>(this.objectClass, cacheDir, this.flushCount, this.version); + } + } +} diff --git a/src/test/java/test/com/qiniu/storage/FormUploadTest.java b/src/test/java/test/com/qiniu/storage/FormUploadTest.java index 35d0756f1..62b14f310 100644 --- a/src/test/java/test/com/qiniu/storage/FormUploadTest.java +++ b/src/test/java/test/com/qiniu/storage/FormUploadTest.java @@ -60,7 +60,7 @@ public void testSyncRetry() { public void testHello2() { TestConfig.TestFile[] files = TestConfig.getTestFileArray(); for (TestConfig.TestFile file : files) { - Configuration config = new Configuration(file.getRegion()); + Configuration config = new Configuration(); config.useHttpsDomains = true; UploadManager uploadManager = new UploadManager(config); hello(uploadManager, file.getBucketName()); diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java new file mode 100644 index 000000000..fa3c10945 --- /dev/null +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -0,0 +1,50 @@ +package test.com.qiniu.util; + +import com.qiniu.util.Cache; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CacheTest { + + @Test + public void testCache() { + Info info = new Info(); + info.foo = "foo"; + info.bar = 1; + + String key = "info_key"; + Cache cache = new Cache.Builder(Info.class) + .setVersion("v1") + .setFlushCount(1) + .builder(); + + cache.cache(key, info); + + + // 1. 测试内存缓存 + Info memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + + // 2. 测试删除内存缓存 + cache.clearMemoryCache(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals(null, memInfo); + + // 3. 测试 load + cache = new Cache.Builder(Info.class) + .setVersion("v1") + .setFlushCount(1) + .builder(); + memInfo = (Info) cache.cacheForKey(key); + assertEquals("foo", memInfo.foo); + } + + static class Info { + String foo; + int bar; + + Info(){ + } + } +} From a73fb1768a58c45a0f737eaeaaa2aa233c07b273 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Thu, 2 Nov 2023 14:37:30 +0800 Subject: [PATCH 08/19] change test case --- src/test/java/test/com/qiniu/util/CacheTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java index fa3c10945..eca8dfa45 100644 --- a/src/test/java/test/com/qiniu/util/CacheTest.java +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -43,8 +43,5 @@ public void testCache() { static class Info { String foo; int bar; - - Info(){ - } } } From 34fbd2c1c2ab8041d51630714691b9ecfb3bebda Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 3 Nov 2023 10:53:45 +0800 Subject: [PATCH 09/19] region create by id handle z0 --- src/main/java/com/qiniu/storage/Region.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/com/qiniu/storage/Region.java b/src/main/java/com/qiniu/storage/Region.java index 35fcb3f86..8765c868c 100644 --- a/src/main/java/com/qiniu/storage/Region.java +++ b/src/main/java/com/qiniu/storage/Region.java @@ -286,6 +286,14 @@ public static Region regionAs0() { * @return Region 实例 */ public static Region createWithRegionId(String regionId) { + if (regionId == null || regionId.isEmpty()) { + return null; + } + + if (regionId.equals("z0")) { + return Region.region0(); + } + return new Builder() .region(regionId) .srcUpHost("up-" + regionId + ".qiniup.com") From a1981208b70bcfafae59848d4381598d708a5557 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 6 Nov 2023 16:14:39 +0800 Subject: [PATCH 10/19] optimize api content-type --- src/main/java/com/qiniu/storage/Api.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/qiniu/storage/Api.java b/src/main/java/com/qiniu/storage/Api.java index b21608395..b43efaf66 100644 --- a/src/main/java/com/qiniu/storage/Api.java +++ b/src/main/java/com/qiniu/storage/Api.java @@ -612,13 +612,12 @@ public StringMap getHeader() throws QiniuException { header.put(key, this.header.get(key)); } - if (header.keySet().contains("Content-Type")) { + if (body == null || body.contentType == null + || header.keySet().contains("Content-Type")) { return header; } - if (body != null) { - header.put("Content-Type", body.contentType.toString()); - } + header.put("Content-Type", body.contentType.toString()); return header; } From e56e19445774d32b7fafa7295fb02cd8d4c46b05 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 6 Nov 2023 16:52:25 +0800 Subject: [PATCH 11/19] optimize test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index ec29f5e96..bccc64288 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -1757,7 +1757,7 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro } long current = new Date().getTime(); - if (current - checkStart > 1000 * 60 * 5.5) { + if (current - checkStart > 1000 * 120 * 5.5) { shouldCheck = false; } From c6c65853fbf50c925ea1dd88416b61682b4097f4 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 16:12:54 +0800 Subject: [PATCH 12/19] cache remove flushcount --- src/main/java/com/qiniu/util/Cache.java | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/qiniu/util/Cache.java b/src/main/java/com/qiniu/util/Cache.java index 0cc297487..bd76e5adc 100644 --- a/src/main/java/com/qiniu/util/Cache.java +++ b/src/main/java/com/qiniu/util/Cache.java @@ -14,9 +14,6 @@ */ public class Cache { - // 当 cache 修改数量到达这个值时,就会 flush,默认是 1 - private final int flushCount; - // 缓存被持久化为一个文件,此文件的文件名为 version,version 默认为:v1 private final String version; @@ -25,14 +22,12 @@ public class Cache { // 内部 private boolean isFlushing = false; - private int needFlushCount = 0; private final ConcurrentHashMap memCache = new ConcurrentHashMap<>(); private final FileRecorder diskCache; - private Cache(Class objectClass, String cacheDir, int flushCount, String version) { + private Cache(Class objectClass, String cacheDir, String version) { this.objectClass = objectClass; - this.flushCount = flushCount; this.version = version; FileRecorder fileRecorder = null; @@ -87,14 +82,7 @@ public void cache(String cacheKey, T object) { } synchronized (this) { - this.needFlushCount++; this.memCache.put(cacheKey, object); - - if (this.needFlushCount < this.flushCount) { - return; - } - - this.needFlushCount = 0; } this.flush(); @@ -141,8 +129,6 @@ public void clearMemoryCache() { } public static class Builder { - // 当 cache 修改数量到达这个值时,就会 flush,默认是 1 - private int flushCount = 1; // 缓存被持久化为一个文件,此文件的文件名为 version,version 默认为:v1 private String version = "v1"; @@ -157,11 +143,6 @@ public Builder(Class objectClass) { this.objectClass = objectClass; } - public Builder setFlushCount(int flushCount) { - this.flushCount = flushCount; - return this; - } - public Builder setCacheDir(String cacheDir) { this.cacheDir = cacheDir; return this; @@ -173,7 +154,7 @@ public Builder setVersion(String version) { } public Cache builder() { - return new Cache<>(this.objectClass, cacheDir, this.flushCount, this.version); + return new Cache<>(this.objectClass, cacheDir, this.version); } } } From 36d4607138d4b659fcf96a802babb4bf0b044727 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 16:25:25 +0800 Subject: [PATCH 13/19] optimize response --- src/main/java/com/qiniu/http/Response.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 00d2da9e5..e09ee5ed8 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -105,7 +105,18 @@ public static Response create(okhttp3.Response response, String address, double error = e.getMessage(); } } + } else if (response.body() != null) { + // 处理其他 body 非预期情况 + if (response.code() >= 300) { + try { + body = response.body().bytes(); + error = new String(body, Constants.UTF_8); + } catch (Exception e) { + error = e.getMessage(); + } + } } + return new Response(response, code, reqId, response.header("X-Log"), via(response), address, duration, error, body); } From cca65c5cf8188f92762eb66968b0908247845449 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 7 Nov 2023 17:42:59 +0800 Subject: [PATCH 14/19] change cache test case --- src/test/java/test/com/qiniu/util/CacheTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/test/com/qiniu/util/CacheTest.java b/src/test/java/test/com/qiniu/util/CacheTest.java index eca8dfa45..9ef3816d5 100644 --- a/src/test/java/test/com/qiniu/util/CacheTest.java +++ b/src/test/java/test/com/qiniu/util/CacheTest.java @@ -16,7 +16,6 @@ public void testCache() { String key = "info_key"; Cache cache = new Cache.Builder(Info.class) .setVersion("v1") - .setFlushCount(1) .builder(); cache.cache(key, info); @@ -34,7 +33,6 @@ public void testCache() { // 3. 测试 load cache = new Cache.Builder(Info.class) .setVersion("v1") - .setFlushCount(1) .builder(); memInfo = (Info) cache.cacheForKey(key); assertEquals("foo", memInfo.foo); From 3fe2c7ab5d7cca21ebee55c95e4aab9aa1d6b9a6 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 14:53:37 +0800 Subject: [PATCH 15/19] region host support contain scheme --- .../java/com/qiniu/storage/ConfigHelper.java | 35 +++++++++++++------ .../java/com/qiniu/storage/UpHostHelper.java | 3 +- src/main/java/com/qiniu/util/UrlUtils.java | 17 +++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/qiniu/storage/ConfigHelper.java b/src/main/java/com/qiniu/storage/ConfigHelper.java index 4858e93a8..cd4ebf21f 100644 --- a/src/main/java/com/qiniu/storage/ConfigHelper.java +++ b/src/main/java/com/qiniu/storage/ConfigHelper.java @@ -39,33 +39,44 @@ public String tryChangeUpHost(String upToken, String lastUsedHost) throws QiniuE private String upHost(String upToken, String lastUsedHost, boolean changeHost, boolean mustReturnUpHost) throws QiniuException { - return getScheme() - + getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + String host = getHelper().upHost(config.region, upToken, UrlUtils.removeHostScheme(lastUsedHost), changeHost, mustReturnUpHost); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getIovipHost(regionReqInfo); + String host = config.region.getIovipHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ioSrcHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return config.region.getIoSrcHost(regionReqInfo); + String host = config.region.getIoSrcHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getApiHost(regionReqInfo); + String host = config.region.getApiHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsHost(regionReqInfo); + String host = config.region.getRsHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsfHost(String ak, String bucket) throws QiniuException { RegionReqInfo regionReqInfo = new RegionReqInfo(ak, bucket); - return getScheme() + config.region.getRsfHost(regionReqInfo); + String host = config.region.getRsfHost(regionReqInfo); + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String rsHost() { @@ -78,7 +89,8 @@ public String rsHost() { if (host == null || host.length() == 0) { host = Configuration.defaultRsHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String apiHost() { @@ -91,7 +103,8 @@ public String apiHost() { if (host == null || host.length() == 0) { host = Configuration.defaultApiHost; } - return getScheme() + host; + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } public String ucHost() { @@ -104,7 +117,9 @@ public String ucHost() { if (host == null || host.length() == 0) { host = Configuration.defaultUcHost; } - return getScheme() + host; + + host = UrlUtils.setHostScheme(host, config.useHttpsDomains); + return host; } List ucHostsWithoutScheme() { diff --git a/src/main/java/com/qiniu/storage/UpHostHelper.java b/src/main/java/com/qiniu/storage/UpHostHelper.java index 67a783624..70f3185e4 100644 --- a/src/main/java/com/qiniu/storage/UpHostHelper.java +++ b/src/main/java/com/qiniu/storage/UpHostHelper.java @@ -47,8 +47,7 @@ String upHost(Region region, String upToken, String lastUsedHost, boolean change regionHostsLRU.put(regionKey, regionHost); } - String host = regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); - return host; + return regionHost.upHost(accHosts, srcHosts, lastUsedHost, changeHost); } private String failedUpHost(String regionKey) { diff --git a/src/main/java/com/qiniu/util/UrlUtils.java b/src/main/java/com/qiniu/util/UrlUtils.java index ceff8acf9..a2892cf8d 100644 --- a/src/main/java/com/qiniu/util/UrlUtils.java +++ b/src/main/java/com/qiniu/util/UrlUtils.java @@ -190,4 +190,21 @@ public static String removeHostScheme(String host) { host = host.replace("https://", ""); return host; } + + + /** + * 如果 host 包含 scheme 则优先使用 host 中包含的 scheme + * 如果 host 不包含 scheme 则按照 useHttps 增加 scheme + */ + public static String setHostScheme(String host, boolean useHttps) { + if (host == null || StringUtils.isNullOrEmpty(host)) { + return null; + } + + if (host.startsWith("http://") || host.startsWith("https://") ) { + return host; + } + + return useHttps ? "https://" : "http://" + host; + } } From cba5907e5f70b9972475397b1b285da422cb6894 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 14 Nov 2023 14:57:56 +0800 Subject: [PATCH 16/19] remove useless code --- src/main/java/com/qiniu/storage/ConfigHelper.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/qiniu/storage/ConfigHelper.java b/src/main/java/com/qiniu/storage/ConfigHelper.java index cd4ebf21f..6b57b396b 100644 --- a/src/main/java/com/qiniu/storage/ConfigHelper.java +++ b/src/main/java/com/qiniu/storage/ConfigHelper.java @@ -166,10 +166,6 @@ List upHostsWithoutScheme() { return new ArrayList<>(hosts); } - private String getScheme() { - return config.useHttpsDomains ? "https://" : "http://"; - } - private void makeSureRegion() { if (config.region == null) { if (config.zone != null) { From 76c8940fe60e12beba46e5263df6671043c77f52 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 11:24:02 +0800 Subject: [PATCH 17/19] handle url setHostScheme --- src/main/java/com/qiniu/util/UrlUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/qiniu/util/UrlUtils.java b/src/main/java/com/qiniu/util/UrlUtils.java index a2892cf8d..f5b8e83e9 100644 --- a/src/main/java/com/qiniu/util/UrlUtils.java +++ b/src/main/java/com/qiniu/util/UrlUtils.java @@ -205,6 +205,6 @@ public static String setHostScheme(String host, boolean useHttps) { return host; } - return useHttps ? "https://" : "http://" + host; + return (useHttps ? "https://" : "http://") + host; } } From a5b22e5f74cb781716f4bbf65be927a81054d0de Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 12:15:19 +0800 Subject: [PATCH 18/19] change download url --- src/main/java/com/qiniu/storage/DownloadUrl.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/qiniu/storage/DownloadUrl.java b/src/main/java/com/qiniu/storage/DownloadUrl.java index 5df7e3735..f2851e679 100644 --- a/src/main/java/com/qiniu/storage/DownloadUrl.java +++ b/src/main/java/com/qiniu/storage/DownloadUrl.java @@ -204,11 +204,7 @@ protected void didBuildUrl() throws QiniuException { } private String getUrlPrefix() throws QiniuException { - if (useHttps) { - return "https://" + domain; - } else { - return "http://" + domain; - } + return UrlUtils.setHostScheme(domain, useHttps); } /** From 4c8963c372ee579eedb0fd41648075c3f5219dde Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 15 Nov 2023 12:18:29 +0800 Subject: [PATCH 19/19] change test case --- src/test/java/test/com/qiniu/storage/BucketTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/test/com/qiniu/storage/BucketTest.java b/src/test/java/test/com/qiniu/storage/BucketTest.java index bccc64288..85634a378 100644 --- a/src/test/java/test/com/qiniu/storage/BucketTest.java +++ b/src/test/java/test/com/qiniu/storage/BucketTest.java @@ -1165,10 +1165,10 @@ public void testFile(TestConfig.TestFile file, BucketManager bucketManager) thro assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode(400))); } try { - testBucketQuota(bucketManager, bucket, 0, 0); - testBucketQuota(bucketManager, bucket, 100, 100); - testBucketQuota(bucketManager, bucket, 0, 100); - testBucketQuota(bucketManager, bucket, 100, -1); +// testBucketQuota(bucketManager, bucket, 0, 0); + testBucketQuota(bucketManager, bucket, 10000, 10000); +// testBucketQuota(bucketManager, bucket, 0, 10000); + testBucketQuota(bucketManager, bucket, 10000, -1); testBucketQuota(bucketManager, bucket, -1, -1); } catch (QiniuException e) { assertTrue(ResCode.find(e.code(), ResCode.getPossibleResCode()));