Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

流上传前获得流的大小 #127

Merged
merged 4 commits into from
Jul 4, 2014
Merged

流上传前获得流的大小 #127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aa3058b
流上传前获得流的大小
sxci Jun 24, 2014
3e337a0
修改另一个测试方法
sxci Jun 24, 2014
b1b82f2
去掉额外的filename
sxci Jul 3, 2014
01760b6
put -> Put
sxci Jul 4, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/qiniu/api/auth/digest/Mac.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public String signRequest(HttpPost post) throws AuthException {
if (entity != null) {
org.apache.http.Header ct = entity.getContentType();
if (ct != null
&& ct.getValue() == "application/x-www-form-urlencoded") {
&& "application/x-www-form-urlencoded".equals(ct.getValue())) {
ByteArrayOutputStream w = new ByteArrayOutputStream();
try {
entity.writeTo(w);
Expand Down
97 changes: 87 additions & 10 deletions src/main/java/com/qiniu/api/io/IoApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
Expand Down Expand Up @@ -33,6 +34,7 @@ private static PutRet put(String uptoken, String key, File file,
return new PutRet(new CallRet(Config.ERROR_CODE, new Exception(
"File does not exist or not readable.")));
}
extra = extra == null ? new PutExtra() : extra;
MultipartEntity requestEntity = new MultipartEntity();
try {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

java.io.File 可以获取到filename

requestEntity.addPart("token", new StringBody(uptoken));
Expand Down Expand Up @@ -85,11 +87,14 @@ private static void setParam(MultipartEntity requestEntity, Map<String, String>
}
}

private static PutRet putStream(String uptoken, String key, InputStream reader,PutExtra extra, String fileName) {
private static PutRet putStream(String uptoken, String key, InputStream reader,
PutExtra extra, long length) {
extra = extra == null ? new PutExtra() : extra;
MultipartEntity requestEntity = new MultipartEntity();
try {
requestEntity.addPart("token", new StringBody(uptoken));
AbstractContentBody inputBody = buildInputStreamBody(reader, extra, fileName != null ? fileName : "null");
String fileName = key != null ? key : "null";
AbstractContentBody inputBody = buildInputStreamBody(reader, extra, fileName, length);
requestEntity.addPart("file", inputBody);
setKey(requestEntity, key);
setParam(requestEntity, extra.params);
Expand All @@ -109,26 +114,97 @@ private static PutRet putStream(String uptoken, String key, InputStream reader,P
return new PutRet(ret);
}

private static InputStreamBody buildInputStreamBody(InputStream reader,PutExtra extra, String fileName){
private static AbstractContentBody buildInputStreamBody(InputStream reader,
PutExtra extra, String fileName, final long length){
if(extra.mimeType != null){
return new InputStreamBody(reader, extra.mimeType, fileName);
return new InputStreamBody(reader, extra.mimeType, fileName){
public long getContentLength() {
return length;
}
};
}else{
return new InputStreamBody(reader, fileName);
return new InputStreamBody(reader, fileName){
public long getContentLength() {
return length;
}
};
}
}

public static PutRet put(String uptoken,String key,InputStream reader,PutExtra extra){
return putStream(uptoken,key,reader,extra, null);

private static PutRet putStream0(String uptoken, String key, InputStream reader,
PutExtra extra, long length){
length = length <= 0 ? getLength(reader) : length;
if(length != -1) {
return putStream(uptoken,key,reader,extra, length);
}else{
return toPutFile(uptoken, key, reader, extra);
}

}

public static PutRet put(String uptoken,String key,InputStream reader,PutExtra extra, String fileName){
return putStream(uptoken,key,reader,extra, fileName);
private static long getLength(InputStream is){
try {
return is.available();
} catch (Exception e) {
return -1;
}
}

private static PutRet toPutFile(String uptoken, String key,
InputStream reader, PutExtra extra) {
File file = null;
try{
file = copyToTmpFile(reader);
return put(uptoken, key, file, extra);
}finally{
if(file != null){
try{file.delete();}catch(Exception e){}
}
}
}


private static File copyToTmpFile(InputStream from){
FileOutputStream os = null;
try{
File to = File.createTempFile("qiniu_", ".tmp");
os = new FileOutputStream(to);
byte[] b = new byte[64 * 1024];
int l;
while ((l = from.read(b)) != -1) {
os.write(b, 0, l);
}
os.flush();
return to;
}catch(Exception e){
throw new RuntimeException("create tmp file failed.", e);
}finally{
if (os != null){
try{os.close();}catch(Exception e){}
}
if (from != null){
try{from.close();}catch(Exception e){}
}
}
}


/**
* @param uptoken
* @param key
* @param reader
* @param extra
* @param length 部分流 is.available() == 0，此时可指定内容长度
* @return
*/
public static PutRet Put(String uptoken,String key,InputStream reader,PutExtra extra, long length){
return putStream0(uptoken,key,reader,extra, length);
}

public static PutRet Put(String uptoken,String key,InputStream reader,PutExtra extra)
{
return put(uptoken,key,reader,extra);
return Put(uptoken,key,reader,extra, -1);
}


Expand Down Expand Up @@ -171,4 +247,5 @@ private static long getCRC32(File file) throws Exception {
}
return crc;
}

}
4 changes: 3 additions & 1 deletion src/main/java/com/qiniu/api/resumableio/StreamSliceUpload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ public StreamSliceUpload(InputStream is,
@Override
protected boolean hasNext() {
try {
return is != null && is.available() > 0;
// 部分流 is.available() == 0，此时通过设定的内容长度判断，
return is != null && (is.available() > 0 || currentBlockIdx * BLOCK_SIZE < contentLength);
} catch (IOException e) {
return false;
}
Expand All @@ -39,6 +40,7 @@ protected boolean hasNext() {
@Override
protected UploadBlock buildNextBlockUpload() throws IOException {
long left = is.available();
left = left > 0 ? left : (contentLength - currentBlockIdx * BLOCK_SIZE);
long start = currentBlockIdx * BLOCK_SIZE;
int len = (int) Math.min(BLOCK_SIZE, left);

Expand Down
4 changes: 4 additions & 0 deletions src/test/java/com/qiniu/testing/HttpClientTimeOutTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ public void testCONNECTION_TIMEOUT() {
Config.SO_TIMEOUT = 20 * 1000;

HttpClient client = Http.getClient();
tearDown();

HttpGet httpget = new HttpGet("http://www.qiniu.com");

s = System.currentTimeMillis();
Expand All @@ -62,6 +64,8 @@ public void testSO_TIMEOUT() {
Config.SO_TIMEOUT = 5;

HttpClient client = Http.getClient();
tearDown();

HttpGet httpget = new HttpGet("http://www.qiniu.com");

s = System.currentTimeMillis();
Expand Down
68 changes: 68 additions & 0 deletions src/test/java/com/qiniu/testing/IOTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
import java.util.Map;
import java.util.UUID;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.json.JSONObject;

import junit.framework.TestCase;
Expand All @@ -14,6 +19,8 @@
import com.qiniu.api.io.IoApi;
import com.qiniu.api.io.PutExtra;
import com.qiniu.api.io.PutRet;
import com.qiniu.api.net.Http;
import com.qiniu.api.resumableio.ResumeableIoApi;
import com.qiniu.api.rs.PutPolicy;
import com.qiniu.api.rs.RSClient;

Expand Down Expand Up @@ -98,6 +105,67 @@ private String getJsonValue(JSONObject jsonObject, String name){
return null;
}
}

public void testNoLengthStream() throws Exception {
PutPolicy p = new PutPolicy(bucketName);
p.returnBody = "{\"key\": $(key), \"hash\": $(etag),\"mimeType\": $(mimeType)}";
String upToken = p.token(mac);

HttpEntity en = getHttpEntity("http://qiniuphotos.qiniudn.com/gogopher.jpg");

class MyInputStream extends InputStream{
InputStream in;
MyInputStream(InputStream is){
this.in = is;
}

@Override
public int read() throws IOException {
// TODO Auto-generated method stub
return in.read();
}

public int available() throws IOException {
throw new IOException();
}

public void close() throws IOException {
in.close();
}

}



PutRet ret = IoApi.Put(upToken, key, new MyInputStream(en.getContent()), null);

System.out.println(ret);
assertTrue(ret.ok());
System.out.println("is.available() = " + en.getContent().available());
}

public void testSetLengthStream() throws Exception {
PutPolicy p = new PutPolicy(bucketName);
p.returnBody = "{\"key\": $(key), \"hash\": $(etag),\"mimeType\": $(mimeType)}";
String upToken = p.token(mac);

HttpEntity en = getHttpEntity("http://qiniuphotos.qiniudn.com/gogopher.jpg");
PutExtra extra = new PutExtra();
extra.mimeType = en.getContentType().getValue();
PutRet ret = IoApi.Put(upToken, key, en.getContent(), extra, en.getContentLength());

System.out.println(ret);
assertTrue(ret.ok());
System.out.println("is.available() = " + en.getContent().available());
}

private HttpEntity getHttpEntity(String url) throws ClientProtocolException, IOException{
HttpClient client = Http.getClient();
HttpGet httpget = new HttpGet(url);
HttpResponse res = client.execute(httpget);
return res.getEntity();
}


@Override
public void tearDown() {
Expand Down
25 changes: 25 additions & 0 deletions src/test/java/com/qiniu/testing/ResumeableioTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
import java.io.IOException;
import java.util.UUID;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.json.JSONException;
import org.json.JSONObject;

Expand All @@ -17,6 +22,7 @@
import com.qiniu.api.config.Config;
import com.qiniu.api.io.PutRet;
import com.qiniu.api.net.CallRet;
import com.qiniu.api.net.Http;
import com.qiniu.api.resumableio.ResumeableIoApi;
import com.qiniu.api.rs.PutPolicy;
import com.qiniu.api.rs.RSClient;
Expand Down Expand Up @@ -108,7 +114,26 @@ private void uploadStream() throws AuthException, JSONException, FileNotFoundExc
assertEquals(mimeType, mt);
}
}

public void testStream() throws Exception {
PutPolicy p = new PutPolicy(bucketName);
p.returnBody = "{\"key\": $(key), \"hash\": $(etag),\"mimeType\": $(mimeType)}";
String upToken = p.token(mac);

HttpEntity en = getHttpEntity("http://qiniuphotos.qiniudn.com/gogopher.jpg");
PutRet ret = ResumeableIoApi.put(en.getContent(), upToken, currentKey, en.getContentType().getValue(), en.getContentLength());

System.out.println(ret);
assertTrue(ret.ok());
System.out.println("is.available() = " + en.getContent().available());
}

private HttpEntity getHttpEntity(String url) throws ClientProtocolException, IOException{
HttpClient client = Http.getClient();
HttpGet httpget = new HttpGet(url);
HttpResponse res = client.execute(httpget);
return res.getEntity();
}

@Override
public void tearDown() {
Expand Down