Skip to content

Commit ee5a5a7

Browse files
author
tonikelope
committed
7.73
CBC-MAC optimization
1 parent 1b076aa commit ee5a5a7

File tree

9 files changed

+113
-108
lines changed

9 files changed

+113
-108
lines changed

‎pom.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>com.tonikelope</groupId>
55
<artifactId>MegaBasterd</artifactId>
6-
<version>7.72</version>
6+
<version>7.73</version>
77
<packaging>jar</packaging>
88
<repositories>
99
<repository>
@@ -20,6 +20,11 @@
2020
</repository>
2121
</repositories>
2222
<dependencies>
23+
<dependency>
24+
<groupId>commons-io</groupId>
25+
<artifactId>commons-io</artifactId>
26+
<version>2.11.0</version>
27+
</dependency>
2328
<dependency>
2429
<groupId>org.sejda.webp-imageio</groupId>
2530
<artifactId>webp-imageio-sejda</artifactId>

‎src/main/java/com/tonikelope/megabasterd/ChunkUploader.java

+24-5
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import java.util.logging.Level;
2626
import java.util.logging.Logger;
2727
import javax.crypto.CipherInputStream;
28+
import org.apache.commons.io.input.QueueInputStream;
29+
import org.apache.commons.io.output.QueueOutputStream;
2830

2931
/**
3032
*
@@ -117,6 +119,8 @@ public void run() {
117119

118120
byte[] buffer = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];
119121

122+
byte[] buffer_enc = new byte[MainPanel.DEFAULT_BYTE_BUFFER_SIZE];
123+
120124
boolean fatal_error = false;
121125

122126
int conta_error = 0;
@@ -141,9 +145,9 @@ public void run() {
141145

142146
chunk_id = _upload.nextChunkId();
143147

144-
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, Upload.CHUNK_SIZE_MULTI);
148+
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
145149

146-
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, Upload.CHUNK_SIZE_MULTI);
150+
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
147151

148152
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
149153

@@ -194,12 +198,25 @@ public void run() {
194198

195199
f.seek(chunk_offset);
196200

197-
try ( CipherInputStream cis = new CipherInputStream(new BufferedInputStream(Channels.newInputStream(f.getChannel())), genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {
201+
ByteArrayOutputStream chunk_mac = new ByteArrayOutputStream();
202+
203+
try ( QueueInputStream qis = new QueueInputStream(); QueueOutputStream qos = qis.newQueueOutputStream(); BufferedInputStream bis = new BufferedInputStream(Channels.newInputStream(f.getChannel())); CipherInputStream cis = new CipherInputStream(qis, genCrypter("AES", "AES/CTR/NoPadding", _upload.getByte_file_key(), forwardMEGALinkKeyIV(_upload.getByte_file_iv(), chunk_offset))); OutputStream out = new ThrottledOutputStream(con.getOutputStream(), _upload.getMain_panel().getStream_supervisor())) {
198204

199205
LOG.log(Level.INFO, "{0} Uploading chunk {1} from worker {2} {3}...", new Object[]{Thread.currentThread().getName(), chunk_id, _id, _upload.getFile_name()});
200206

201-
while (!_exit && tot_bytes_up < chunk_size && (reads = cis.read(buffer)) != -1) {
202-
out.write(buffer, 0, reads);
207+
while (!_exit && tot_bytes_up < chunk_size && (reads = bis.read(buffer)) != -1) {
208+
209+
chunk_mac.write(buffer, 0, reads);
210+
211+
for (int i = 0; i < reads; i++) {
212+
qos.write(buffer[i]);
213+
}
214+
215+
for (int i = 0; i < reads; i++) {
216+
buffer_enc[i] = (byte) cis.read();
217+
}
218+
219+
out.write(buffer_enc, 0, reads);
203220

204221
_upload.getPartialProgress().add((long) reads);
205222

@@ -215,6 +232,8 @@ public void run() {
215232

216233
}
217234
}
235+
236+
_upload.getMac_generator().CHUNK_QUEUE.put(chunk_offset, chunk_mac);
218237
}
219238

220239
if (!_exit) {

‎src/main/java/com/tonikelope/megabasterd/MainPanel.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
*/
6868
public final class MainPanel {
6969

70-
public static final String VERSION = "7.72";
70+
public static final String VERSION = "7.73";
7171
public static final boolean FORCE_SMART_PROXY = false; //TRUE FOR DEBUGING SMART PROXY
7272
public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
7373
public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;

‎src/main/java/com/tonikelope/megabasterd/MiscTools.java

+16-39
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.io.BufferedInputStream;
3131
import java.io.ByteArrayInputStream;
3232
import java.io.ByteArrayOutputStream;
33+
import java.io.DataOutputStream;
3334
import java.io.File;
3435
import java.io.FileInputStream;
3536
import java.io.IOException;
@@ -48,6 +49,7 @@
4849
import java.net.URL;
4950
import java.net.URLDecoder;
5051
import java.nio.ByteBuffer;
52+
import java.nio.ByteOrder;
5153
import java.nio.IntBuffer;
5254
import java.nio.file.DirectoryStream;
5355
import java.nio.file.Files;
@@ -239,54 +241,29 @@ public static void setNimbusLookAndFeel() {
239241
public static int[] bin2i32a(byte[] bin) {
240242
int l = (int) (4 * Math.ceil((double) bin.length / 4));
241243

242-
byte[] new_bin = Arrays.copyOfRange(bin, 0, l);
244+
IntBuffer intBuf = ByteBuffer.wrap(bin, 0, l).order(ByteOrder.BIG_ENDIAN).asIntBuffer();
243245

244-
bin = new_bin;
246+
int[] array = new int[intBuf.remaining()];
245247

246-
ByteBuffer bin_buffer = ByteBuffer.wrap(bin);
247-
IntBuffer int_buffer = bin_buffer.asIntBuffer();
248+
intBuf.get(array);
248249

249-
if (int_buffer.hasArray()) {
250-
return int_buffer.array();
251-
} else {
252-
ArrayList<Integer> list = new ArrayList<>();
253-
254-
while (int_buffer.hasRemaining()) {
255-
list.add(int_buffer.get());
256-
}
257-
258-
int[] aux = new int[list.size()];
259-
260-
for (int i = 0; i < aux.length; i++) {
261-
aux[i] = list.get(i);
262-
}
263-
264-
return aux;
265-
}
250+
return array;
266251
}
267252

268-
public static byte[] i32a2bin(int[] i32a) {
269-
ByteBuffer bin_buffer = ByteBuffer.allocate(i32a.length * 4);
270-
IntBuffer int_buffer = bin_buffer.asIntBuffer();
271-
int_buffer.put(i32a);
253+
public static byte[] i32a2bin(int[] values) {
254+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
272255

273-
if (bin_buffer.hasArray()) {
274-
return bin_buffer.array();
275-
} else {
276-
ArrayList<Byte> list = new ArrayList<>();
277-
278-
while (int_buffer.hasRemaining()) {
279-
list.add(bin_buffer.get());
280-
}
281-
282-
byte[] aux = new byte[list.size()];
256+
DataOutputStream dos = new DataOutputStream(baos);
283257

284-
for (int i = 0; i < aux.length; i++) {
285-
aux[i] = list.get(i);
258+
for (int i = 0; i < values.length; ++i) {
259+
try {
260+
dos.writeInt(values[i]);
261+
} catch (IOException ex) {
262+
Logger.getLogger(MiscTools.class.getName()).log(Level.SEVERE, null, ex);
286263
}
287-
288-
return aux;
289264
}
265+
266+
return baos.toByteArray();
290267
}
291268

292269
public static BigInteger mpi2big(byte[] s) {

‎src/main/java/com/tonikelope/megabasterd/Upload.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
public class Upload implements Transference, Runnable, SecureSingleThreadNotifiable {
4040

4141
public static final int WORKERS_DEFAULT = 6;
42-
public static final int CHUNK_SIZE_MULTI = 1; //Otra cosa da errores al reanudar una subida (investigar)
4342
public static final boolean DEFAULT_THUMBNAILS = true;
4443
public static final boolean UPLOAD_LOG = false;
4544
private static final Logger LOG = Logger.getLogger(Upload.class.getName());
@@ -62,7 +61,7 @@ public class Upload implements Transference, Runnable, SecureSingleThreadNotifia
6261
private long _last_chunk_id_dispatched;
6362
private final ConcurrentLinkedQueue<Long> _partialProgressQueue;
6463
private final ExecutorService _thread_pool;
65-
private int[] _file_meta_mac;
64+
private volatile int[] _file_meta_mac;
6665
private String _fid;
6766
private boolean _notified;
6867
private volatile String _completion_handler;
@@ -1305,7 +1304,7 @@ public boolean isStatusError() {
13051304
public long calculateLastUploadedChunk(long bytes_read) {
13061305

13071306
if (bytes_read > 3584 * 1024) {
1308-
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * Upload.CHUNK_SIZE_MULTI));
1307+
return 7 + (long) Math.floor((float) (bytes_read - 3584 * 1024) / (1024 * 1024 * 1));
13091308
} else {
13101309
long i = 0, tot = 0;
13111310

‎src/main/java/com/tonikelope/megabasterd/UploadMACGenerator.java

+51-45
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111

1212
import static com.tonikelope.megabasterd.CryptTools.*;
1313
import static com.tonikelope.megabasterd.MiscTools.*;
14-
import java.io.BufferedInputStream;
15-
import java.io.FileInputStream;
16-
import java.io.IOException;
14+
import java.io.ByteArrayInputStream;
15+
import java.io.ByteArrayOutputStream;
1716
import java.util.HashMap;
17+
import java.util.concurrent.ConcurrentHashMap;
1818
import java.util.logging.Level;
1919
import java.util.logging.Logger;
2020
import javax.crypto.BadPaddingException;
@@ -33,6 +33,7 @@ public class UploadMACGenerator implements Runnable, SecureSingleThreadNotifiabl
3333
private final Object _secure_notify_lock;
3434
private boolean _notified;
3535
private volatile boolean _exit;
36+
public final ConcurrentHashMap<Long, ByteArrayOutputStream> CHUNK_QUEUE = new ConcurrentHashMap<>();
3637

3738
public UploadMACGenerator(Upload upload) {
3839
_secure_notify_lock = new Object();
@@ -122,36 +123,40 @@ public void run() {
122123

123124
Cipher cryptor = genCrypter("AES", "AES/CBC/NoPadding", _upload.getByte_file_key(), i32a2bin(mac_iv));
124125

125-
try ( BufferedInputStream is = new BufferedInputStream(new FileInputStream(_upload.getFile_name()))) {
126+
int[] chunk_mac = new int[4];
127+
byte[] byte_block = new byte[16];
128+
byte[] chunk_bytes;
126129

127-
if (tot > 0) {
128-
is.skip(tot);
129-
}
130+
try {
131+
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {
130132

131-
int[] chunk_mac = new int[4];
132-
byte[] byte_block = new byte[16];
133+
int reads;
133134

134-
try {
135-
while (!_exit && !_upload.isStopped() && !_upload.getMain_panel().isExit()) {
135+
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
136136

137-
int reads;
137+
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
138138

139-
long chunk_offset = ChunkWriterManager.calculateChunkOffset(chunk_id, 1);
139+
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
140140

141-
long chunk_size = ChunkWriterManager.calculateChunkSize(chunk_id, _upload.getFile_size(), chunk_offset, 1);
141+
while (!CHUNK_QUEUE.containsKey(chunk_offset)) {
142+
MiscTools.pausar(1000);
143+
}
142144

143-
ChunkWriterManager.checkChunkID(chunk_id, _upload.getFile_size(), chunk_offset);
145+
try {
144146

145-
try {
147+
chunk_mac[0] = file_iv[0];
148+
chunk_mac[1] = file_iv[1];
149+
chunk_mac[2] = file_iv[0];
150+
chunk_mac[3] = file_iv[1];
146151

147-
chunk_mac[0] = file_iv[0];
148-
chunk_mac[1] = file_iv[1];
149-
chunk_mac[2] = file_iv[0];
150-
chunk_mac[3] = file_iv[1];
152+
long conta_chunk = 0L;
151153

152-
long conta_chunk = 0L;
154+
try ( ByteArrayOutputStream baos = CHUNK_QUEUE.remove(chunk_offset)) {
155+
chunk_bytes = baos.toByteArray();
156+
}
153157

154-
while (conta_chunk < chunk_size && (reads = is.read(byte_block)) != -1) {
158+
try ( ByteArrayInputStream bais = new ByteArrayInputStream(chunk_bytes)) {
159+
while (conta_chunk < chunk_size && (reads = bais.read(byte_block)) != -1) {
155160

156161
if (reads < byte_block.length) {
157162
for (int i = reads; i < byte_block.length; i++) {
@@ -172,45 +177,46 @@ public void run() {
172177
tot += reads;
173178

174179
}
180+
}
175181

176-
for (int i = 0; i < file_mac.length; i++) {
177-
file_mac[i] ^= chunk_mac[i];
178-
}
182+
for (int i = 0; i < file_mac.length; i++) {
183+
file_mac[i] ^= chunk_mac[i];
184+
}
179185

180-
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
186+
file_mac = bin2i32a(cryptor.doFinal(i32a2bin(file_mac)));
181187

182-
} catch (IOException | IllegalBlockSizeException | BadPaddingException ex) {
183-
LOG.log(Level.SEVERE, ex.getMessage());
184-
}
188+
} catch (IllegalBlockSizeException | BadPaddingException ex) {
189+
LOG.log(Level.SEVERE, ex.getMessage());
190+
}
185191

186-
chunk_id++;
192+
chunk_id++;
187193

188-
int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);
194+
int new_cbc_per = (int) ((((double) tot) / _upload.getFile_size()) * 100);
189195

190-
if (new_cbc_per != cbc_per) {
191-
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
192-
cbc_per = new_cbc_per;
193-
}
196+
if (new_cbc_per != cbc_per) {
197+
_upload.getView().updateCBC("CBC-MAC " + String.valueOf(new_cbc_per) + "%");
198+
cbc_per = new_cbc_per;
194199
}
195200

196-
mac = (tot == _upload.getFile_size());
201+
}
197202

198-
} catch (ChunkInvalidException e) {
203+
mac = (tot == _upload.getFile_size());
199204

200-
mac = true;
201-
}
205+
} catch (ChunkInvalidException e) {
202206

203-
_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));
207+
mac = true;
208+
}
204209

205-
if (mac) {
210+
_upload.setTemp_mac_data(String.valueOf(tot) + "#" + String.valueOf(chunk_id) + "#" + Bin2BASE64(i32a2bin(file_mac)));
206211

207-
int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};
212+
if (mac) {
208213

209-
_upload.setFile_meta_mac(meta_mac);
214+
int[] meta_mac = {file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]};
210215

211-
LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});
216+
_upload.setFile_meta_mac(meta_mac);
217+
218+
LOG.log(Level.INFO, "{0} MAC GENERATOR {1} finished MAC CALCULATION. Waiting workers to finish uploading (if any)...", new Object[]{Thread.currentThread().getName(), getUpload().getFile_name()});
212219

213-
}
214220
}
215221

216222
while (!_exit && !_upload.isStopped() && !_upload.getChunkworkers().isEmpty()) {

‎src/main/java/com/tonikelope/megabasterd/UploadView.form

+7-8
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,13 @@
106106
<Component id="slot_status_label" min="-2" max="-2" attributes="0"/>
107107
</Group>
108108
<EmptySpace min="-2" max="-2" attributes="0"/>
109-
<Group type="103" groupAlignment="3" attributes="0">
110-
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
111-
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
112-
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
113-
<Component id="open_browser_button" alignment="0" max="32767" attributes="0"/>
109+
<Group type="103" groupAlignment="0" attributes="0">
110+
<Component id="open_browser_button" alignment="0" max="-2" attributes="0"/>
111+
<Group type="103" groupAlignment="3" attributes="0">
112+
<Component id="folder_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
113+
<Component id="file_link_button" alignment="3" min="-2" max="-2" attributes="0"/>
114+
<Component id="file_size_label" alignment="3" min="-2" max="-2" attributes="0"/>
115+
</Group>
114116
</Group>
115117
<EmptySpace min="-2" max="-2" attributes="0"/>
116118
<Component id="progress_pbar" min="-2" max="-2" attributes="0"/>
@@ -387,9 +389,6 @@
387389
<Image iconType="3" name="/images/icons8-export-30.png"/>
388390
</Property>
389391
<Property name="text" type="java.lang.String" value="Open folder in browser"/>
390-
<Property name="cursor" type="java.awt.Cursor" editor="org.netbeans.modules.form.editors2.CursorEditor">
391-
<Color id="Cursor de Mano"/>
392-
</Property>
393392
<Property name="doubleBuffered" type="boolean" value="true"/>
394393
</Properties>
395394
<Events>

0 commit comments

Comments
 (0)