Skip to content

Commit 82b84af

Browse files
author
tonikelope
committed
2.25
-Upload multi-slot multi-size remove
1 parent 96520d7 commit 82b84af

10 files changed

+64
-84
lines changed

‎src/megabasterd/ChunkDownloader.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ public void run() {
9393
String worker_url = null;
9494
Chunk chunk;
9595
int reads, conta_error, http_status;
96-
byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
9796
InputStream is;
9897
boolean error;
9998

@@ -110,7 +109,7 @@ public void run() {
110109
worker_url = _download.getDownloadUrlForWorker();
111110
}
112111

113-
chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI);
112+
chunk = new Chunk(_download.nextChunkId(), _download.getFile_size(), worker_url, Download.CHUNK_SIZE_MULTI);
114113

115114
HttpGet httpget = new HttpGet(new URI(chunk.getUrl()));
116115

@@ -131,6 +130,8 @@ public void run() {
131130

132131
} else {
133132

133+
byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
134+
134135
while (!_exit && !_download.isStopped() && !_download.getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer)) != -1) {
135136

136137
chunk.getOutputStream().write(buffer, 0, reads);

‎src/megabasterd/ChunkDownloaderMono.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,7 @@ public ChunkDownloaderMono(Download download) {
2828
public void run() {
2929
String worker_url = null;
3030
Chunk chunk;
31-
int reads, max_reads, conta_error, http_status = 200;
32-
byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
31+
int reads, conta_error, http_status = 200;
3332
boolean error;
3433
HttpGet httpget = null;
3534
CloseableHttpResponse httpresponse = null;
@@ -93,7 +92,9 @@ public void run() {
9392

9493
if (!isExit() && !getDownload().isStopped() && is != null) {
9594

96-
while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, (max_reads = (int) (chunk.getSize() - chunk.getOutputStream().size())) <= buffer.length ? max_reads : buffer.length)) != -1) {
95+
byte[] buffer = new byte[THROTTLE_SLICE_SIZE];
96+
97+
while (!getDownload().isStopped() && !getDownload().getChunkwriter().isExit() && chunk.getOutputStream().size() < chunk.getSize() && (reads = is.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length))) != -1) {
9798
chunk.getOutputStream().write(buffer, 0, reads);
9899

99100
getDownload().getPartialProgressQueue().add(reads);

‎src/megabasterd/ChunkUploader.java

+6-5
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,7 @@ public void run() {
109109

110110
String worker_url = _upload.getUl_url();
111111
Chunk chunk;
112-
int reads, to_read, conta_error, re, http_status, tot_bytes_up;
113-
byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
112+
int reads, conta_error, re, http_status, tot_bytes_up;
114113
boolean error;
115114
OutputStream out;
116115

@@ -119,14 +118,15 @@ public void run() {
119118
conta_error = 0;
120119

121120
while (!_exit && !_upload.isStopped()) {
122-
chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url, Transference.CHUNK_SIZE_MULTI);
121+
chunk = new Chunk(_upload.nextChunkId(), _upload.getFile_size(), worker_url);
123122

124123
f.seek(chunk.getOffset());
125124

125+
byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
126+
126127
do {
127-
to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size());
128128

129-
re = f.read(buffer, 0, to_read);
129+
re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length));
130130

131131
chunk.getOutputStream().write(buffer, 0, re);
132132

@@ -164,6 +164,7 @@ public CloseableHttpResponse call() throws IOException {
164164
THREAD_POOL.execute(futureTask);
165165
out = new ThrottledOutputStream(pipeout, _upload.getMain_panel().getStream_supervisor());
166166
System.out.println(" Subiendo chunk " + chunk.getId() + " desde worker " + _id + "...");
167+
System.out.println(chunk.getUrl());
167168
while (!_exit && !_upload.isStopped() && (reads = cis.read(buffer)) != -1) {
168169
out.write(buffer, 0, reads);
169170

‎src/megabasterd/ChunkUploaderMono.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,7 @@ public void run() {
5050

5151
String worker_url = getUpload().getUl_url();
5252
Chunk chunk;
53-
int reads, to_read, conta_error, re, http_status, tot_bytes_up = -1;
54-
byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
53+
int reads, conta_error, re, http_status, tot_bytes_up = -1;
5554
boolean error = false;
5655

5756
try (CloseableHttpClient httpclient = MiscTools.getApacheKissHttpClient(); RandomAccessFile f = new RandomAccessFile(getUpload().getFile_name(), "r");) {
@@ -63,16 +62,18 @@ public void run() {
6362
FutureTask<CloseableHttpResponse> futureTask = null;
6463

6564
while (!isExit() && !getUpload().isStopped()) {
65+
6666
CloseableHttpResponse httpresponse = null;
6767

6868
chunk = new Chunk(getUpload().nextChunkId(), getUpload().getFile_size(), null);
6969

7070
f.seek(chunk.getOffset());
7171

72+
byte[] buffer = new byte[MainPanel.THROTTLE_SLICE_SIZE];
73+
7274
do {
73-
to_read = chunk.getSize() - chunk.getOutputStream().size() >= buffer.length ? buffer.length : (int) (chunk.getSize() - chunk.getOutputStream().size());
7475

75-
re = f.read(buffer, 0, to_read);
76+
re = f.read(buffer, 0, Math.min((int) (chunk.getSize() - chunk.getOutputStream().size()), buffer.length));
7677

7778
chunk.getOutputStream().write(buffer, 0, re);
7879

‎src/megabasterd/ChunkWriter.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public void run() {
152152

153153
private long calculateLastWrittenChunk(long temp_file_size) {
154154
if (temp_file_size > 3584 * 1024) {
155-
return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * Transference.CHUNK_SIZE_MULTI));
155+
return 7 + (long) Math.ceil((temp_file_size - 3584 * 1024) / (1024 * 1024 * (_download.isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1)));
156156
} else {
157157
int i = 0, tot = 0;
158158

‎src/megabasterd/Download.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public final class Download implements Transference, Runnable, SecureSingleThrea
5959
public static final boolean USE_SLOTS_DEFAULT = false;
6060
public static final int WORKERS_DEFAULT = 6;
6161
public static final boolean USE_MEGA_ACCOUNT_DOWN = false;
62+
public static final int CHUNK_SIZE_MULTI = 10;
6263

6364
private final MainPanel _main_panel;
6465
private volatile DownloadView _view = null; //lazy init
@@ -629,7 +630,7 @@ public void run() {
629630
} else {
630631
getView().hideAllExceptStatus();
631632

632-
exit_message = "OOOPS!! Something (bad) happened but... what?";
633+
exit_message = "OOOPS!! Something (bad) happened but... what? Progress: " + String.valueOf(_progress) + " File size: " + String.valueOf(_file_size);
633634

634635
getView().printStatusError(exit_message);
635636

@@ -1162,7 +1163,7 @@ public void emergencyStopDownloader(String reason) {
11621163

11631164
public long calculateMaxTempFileSize(long size) {
11641165
if (size > 3584 * 1024) {
1165-
long reminder = (size - 3584 * 1024) % (1024 * 1024 * Transference.CHUNK_SIZE_MULTI);
1166+
long reminder = (size - 3584 * 1024) % (1024 * 1024 * (isUse_slots() ? Download.CHUNK_SIZE_MULTI : 1));
11661167

11671168
return reminder == 0 ? size : (size - reminder);
11681169
} else {

‎src/megabasterd/MainPanel.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
*/
5858
public final class MainPanel {
5959

60-
public static final String VERSION = "2.24";
60+
public static final String VERSION = "2.25";
6161
public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
6262
public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;
6363
public static final int STREAMER_PORT = 1337;

‎src/megabasterd/Transference.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,12 @@
99
public interface Transference {
1010

1111
int MIN_WORKERS = 2;
12-
int MAX_WORKERS = 20;
12+
int MAX_WORKERS = 30;
1313
int MAX_SIM_TRANSFERENCES = 20;
1414
int SIM_TRANSFERENCES_DEFAULT = 2;
1515
boolean LIMIT_TRANSFERENCE_SPEED_DEFAULT = false;
1616
int MAX_TRANSFERENCE_SPEED_DEFAULT = 5;
1717
int MAX_WAIT_WORKERS_SHUTDOWN = 15;
18-
int CHUNK_SIZE_MULTI = 10;
1918
Integer[] FATAL_ERROR_API_CODES = {-2, -8, -9, -10, -11, -12, -13, -14, -15, -16, 22, 23, 24, 25};
2019

2120
void start();

‎src/megabasterd/Upload.java

+18-3
Original file line numberDiff line numberDiff line change
@@ -381,11 +381,11 @@ public void provisionIt() {
381381

382382
String[] fdata = new String(data).split("\\|");
383383

384-
_last_chunk_id_dispatched = Long.parseLong(fdata[0]);
384+
_progress = Long.parseLong(fdata[0]);
385385

386-
_progress = Long.parseLong(fdata[1]);
386+
_last_chunk_id_dispatched = calculateLastUploadedChunk(_progress);
387387

388-
_saved_file_mac = bin2i32a(BASE642Bin(fdata[2]));
388+
_saved_file_mac = bin2i32a(BASE642Bin(fdata[1]));
389389

390390
} else if (temp_file.exists()) {
391391

@@ -1044,4 +1044,19 @@ public boolean isStatusError() {
10441044
return _status_error;
10451045
}
10461046

1047+
public long calculateLastUploadedChunk(long bytes_read) {
1048+
if (bytes_read > 3584 * 1024) {
1049+
return 7 + (long) Math.ceil((bytes_read - 3584 * 1024) / (1024 * 1024));
1050+
} else {
1051+
int i = 0, tot = 0;
1052+
1053+
while (tot < bytes_read) {
1054+
i++;
1055+
tot += i * 128 * 1024;
1056+
}
1057+
1058+
return i;
1059+
}
1060+
}
1061+
10471062
}

‎src/megabasterd/UploadMACGenerator.java

+22-61
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public void run() {
105105
Chunk chunk;
106106
int[] file_iv = bin2i32a(_upload.getByte_file_iv()), int_block, file_mac = _upload.getSaved_file_mac(), mac_iv = CryptTools.AES_ZERO_IV_I32A;
107107
int reads;
108-
byte[] byte_block = new byte[16];
108+
109109
String temp_file_data;
110110
boolean new_chunk = false;
111111
boolean upload_workers_finish = false;
@@ -130,89 +130,50 @@ public void run() {
130130

131131
try {
132132

133-
if (chunk.getId() <= 7 || !_upload.isUse_slots()) {
134-
135-
int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
133+
int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
134+
byte[] byte_block = new byte[16];
136135

137-
while ((reads = chunk_is.read(byte_block)) != -1) {
136+
while ((reads = chunk_is.read(byte_block)) != -1) {
138137

139-
if (reads < byte_block.length) {
140-
for (int i = reads; i < byte_block.length; i++) {
141-
byte_block[i] = 0;
142-
}
138+
if (reads < byte_block.length) {
139+
for (int i = reads; i < byte_block.length; i++) {
140+
byte_block[i] = 0;
143141
}
144-
145-
int_block = bin2i32a(byte_block);
146-
147-
for (int i = 0; i < chunk_mac.length; i++) {
148-
chunk_mac[i] ^= int_block[i];
149-
}
150-
151-
chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
152142
}
153143

154-
for (int i = 0; i < file_mac.length; i++) {
155-
file_mac[i] ^= chunk_mac[i];
156-
}
157-
158-
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
159-
160-
_bytes_read += chunk.getSize();
161-
162-
} else {
163-
do {
164-
int[] chunk_mac = {file_iv[0], file_iv[1], file_iv[0], file_iv[1]};
165-
166-
long chunk_size = 0;
167-
168-
do {
169-
170-
if ((reads = chunk_is.read(byte_block)) != -1) {
171-
if (reads < byte_block.length) {
172-
for (int i = reads; i < byte_block.length; i++) {
173-
byte_block[i] = 0;
174-
}
175-
}
144+
int_block = bin2i32a(byte_block);
176145

177-
int_block = bin2i32a(byte_block);
178-
179-
for (int i = 0; i < chunk_mac.length; i++) {
180-
chunk_mac[i] ^= int_block[i];
181-
}
182-
183-
chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
184-
185-
chunk_size += reads;
186-
}
187-
} while (reads != -1 && chunk_size < 1024 * 1024);
188-
189-
for (int i = 0; i < file_mac.length; i++) {
190-
file_mac[i] ^= chunk_mac[i];
191-
}
192-
193-
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
146+
for (int i = 0; i < chunk_mac.length; i++) {
147+
chunk_mac[i] ^= int_block[i];
148+
}
194149

195-
_bytes_read += chunk_size;
150+
chunk_mac = bin2i32a(cryptor.doFinal(i32a2bin(chunk_mac)));
151+
}
196152

197-
} while (reads != -1);
153+
for (int i = 0; i < file_mac.length; i++) {
154+
file_mac[i] ^= chunk_mac[i];
198155
}
199156

157+
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
158+
200159
} catch (IOException | IllegalBlockSizeException | BadPaddingException ex) {
201160
getLogger(UploadMACGenerator.class.getName()).log(Level.SEVERE, null, ex);
202161
}
203162

204-
_chunk_queue.remove(chunk.getId());
163+
_bytes_read += chunk.getSize();
205164

206165
_last_chunk_id_read = chunk.getId();
207166

167+
_chunk_queue.remove(chunk.getId());
168+
208169
new_chunk = true;
209170
}
210171

211172
if (!upload_workers_finish && new_chunk) {
212173

213-
temp_file_data = (String.valueOf(_last_chunk_id_read) + "|" + String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac)));
174+
temp_file_data = (String.valueOf(_bytes_read) + "|" + Bin2BASE64(i32a2bin(file_mac)));
214175

215-
System.out.println("Macgenerator -> " + temp_file_data);
176+
System.out.println("Macgenerator -> " + temp_file_data + " " + _upload.calculateLastUploadedChunk(_bytes_read) + " " + _last_chunk_id_read);
216177

217178
temp_file_out = new FileOutputStream(temp_file);
218179

0 commit comments

Comments
 (0)