Skip to content

Commit 27dcf80

Browse files
daniel-werfm3philippotto
authored
Use new zip.js version to allow zip64 uploads (#6939)
* Reject dataset uploads if organization storage quota is exceeded * warn user in upload-dataset-view when storage is exceeded and disable upload button * changelog * Use new zip.js version to allow zip64 uploads, for example * update changelog * also notify airbrake for zip errors * Remove jszip dependency and rewrite code to use zip.js instead --------- Co-authored-by: Florian M <[email protected]> Co-authored-by: Philipp Otto <[email protected]> Co-authored-by: Philipp Otto <[email protected]>
1 parent d46eb37 commit 27dcf80

File tree

5 files changed

+63
-84
lines changed

5 files changed

+63
-84
lines changed

‎CHANGELOG.unreleased.md

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
1515
- Added list of all respective team members to the administration page for teams. [#6915](https://github.com/scalableminds/webknossos/pull/6915)
1616
- Added email notifications for WK worker jobs. [#6918](https://github.com/scalableminds/webknossos/pull/6918)
1717
- Added support for viewing sharded neuroglancer precomputed datasets. [#6920](https://github.com/scalableminds/webknossos/pull/6920)
18+
- Added support for uploading zip64 files. [#6939](https://github.com/scalableminds/webknossos/pull/6939)
1819

1920
### Changed
2021
- Interpolation during rendering is now more performance intensive, since the rendering approach was changed. Therefore, interpolation is disabled by default. On the flip side, the rendered quality is often higher than it used to be. [#6748](https://github.com/scalableminds/webknossos/pull/6748)

‎frontend/javascripts/admin/dataset/dataset_upload_view.tsx

+33-35
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ import Toast from "libs/toast";
3232
import * as Utils from "libs/utils";
3333
import messages from "messages";
3434
import { trackAction } from "oxalis/model/helpers/analytics";
35-
// @ts-expect-error ts-migrate(2306) FIXME: File ... Remove this comment to see the full error message
36-
import { createReader, BlobReader, ZipReader, Entry } from "zip-js-webpack";
35+
import { BlobReader, ZipReader, Entry } from "@zip.js/zip.js";
3736
import {
3837
CardContainer,
3938
DatasetNameFormItem,
@@ -473,7 +472,7 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
473472
);
474473
};
475474

476-
validateFiles = (files: FileWithPath[]) => {
475+
validateFiles = async (files: FileWithPath[]) => {
477476
if (files.length === 0) {
478477
return;
479478
}
@@ -490,41 +489,40 @@ class DatasetUploadView extends React.Component<PropsWithFormAndRouter, State> {
490489
});
491490

492491
if (fileExtension === "zip") {
493-
createReader(
494-
new BlobReader(file),
495-
(reader: ZipReader) => {
496-
reader.getEntries((entries: Array<Entry>) => {
497-
const wkwFile = entries.find((entry: Entry) =>
498-
Utils.isFileExtensionEqualTo(entry.filename, "wkw"),
499-
);
500-
const needsConversion = wkwFile == null;
501-
this.handleNeedsConversionInfo(needsConversion);
502-
503-
const nmlFile = entries.find((entry: Entry) =>
504-
Utils.isFileExtensionEqualTo(entry.filename, "nml"),
505-
);
506-
if (nmlFile) {
507-
Modal.error({
508-
content: messages["dataset.upload_zip_with_nml"],
509-
});
510-
}
511-
});
512-
},
513-
() => {
492+
try {
493+
const reader = new ZipReader(new BlobReader(file));
494+
const entries = await reader.getEntries();
495+
await reader.close();
496+
const wkwFile = entries.find((entry: Entry) =>
497+
Utils.isFileExtensionEqualTo(entry.filename, "wkw"),
498+
);
499+
const needsConversion = wkwFile == null;
500+
this.handleNeedsConversionInfo(needsConversion);
501+
502+
const nmlFile = entries.find((entry: Entry) =>
503+
Utils.isFileExtensionEqualTo(entry.filename, "nml"),
504+
);
505+
if (nmlFile) {
514506
Modal.error({
515-
content: messages["dataset.upload_invalid_zip"],
507+
content: messages["dataset.upload_zip_with_nml"],
516508
});
517-
const form = this.formRef.current;
518-
519-
if (!form) {
520-
return;
521-
}
509+
}
510+
} catch (e) {
511+
console.error(e);
512+
ErrorHandling.notify(e as Error);
513+
Modal.error({
514+
content: messages["dataset.upload_invalid_zip"],
515+
});
516+
const form = this.formRef.current;
517+
518+
if (!form) {
519+
return;
520+
}
522521

523-
form.setFieldsValue({
524-
zipFile: [],
525-
});
526-
},
527-
);
522+
form.setFieldsValue({
523+
zipFile: [],
524+
});
525+
}
528526
// We return here since not more than 1 zip archive is supported anyway.
529527
return;
530528
} else if (fileExtension === "wkw") {

‎frontend/javascripts/oxalis/view/right-border-tabs/skeleton_tab_view.tsx

+21-13
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
import { batchActions } from "redux-batched-actions";
2222
import { connect } from "react-redux";
2323
import { saveAs } from "file-saver";
24-
import JSZip from "jszip";
24+
import { BlobReader, BlobWriter, ZipReader, Entry } from "@zip.js/zip.js";
2525
import * as React from "react";
2626
import _ from "lodash";
2727
import memoizeOne from "memoize-one";
@@ -208,22 +208,29 @@ export async function importTracingFiles(files: Array<File>, createGroupForEachF
208208

209209
const tryParsingFileAsZip = async (file: File) => {
210210
try {
211-
// @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'Promise<ArrayBuffer>' is not ass... Remove this comment to see the full error message
212-
const zipFile = await JSZip().loadAsync(readFileAsArrayBuffer(file));
213-
const nmlFileName = Object.keys(zipFile.files).find((key) =>
214-
Utils.isFileExtensionEqualTo(key, "nml"),
211+
const reader = new ZipReader(new BlobReader(file));
212+
const entries = await reader.getEntries();
213+
const nmlFileEntry = entries.find((entry: Entry) =>
214+
Utils.isFileExtensionEqualTo(entry.filename, "nml"),
215215
);
216-
// @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
217-
const nmlFile = await zipFile.file(nmlFileName).async("blob");
216+
217+
if (nmlFileEntry == null) {
218+
await reader.close();
219+
throw Error("Zip file doesn't contain an NML file.");
220+
}
221+
222+
const nmlBlob = await nmlFileEntry.getData(new BlobWriter());
223+
const nmlFile = new File([nmlBlob], nmlFileEntry.filename);
224+
218225
const nmlImportActions = await tryParsingFileAsNml(nmlFile);
219-
const dataFileName = Object.keys(zipFile.files).find((key) =>
220-
Utils.isFileExtensionEqualTo(key, "zip"),
226+
227+
const dataFileEntry = entries.find((entry: Entry) =>
228+
Utils.isFileExtensionEqualTo(entry.filename, "zip"),
221229
);
222230

223-
if (dataFileName) {
224-
// @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
225-
const dataBlob = await zipFile.file(dataFileName).async("blob");
226-
const dataFile = new File([dataBlob], dataFileName);
231+
if (dataFileEntry) {
232+
const dataBlob = await dataFileEntry.getData(new BlobWriter());
233+
const dataFile = new File([dataBlob], dataFileEntry.filename);
227234
await Model.ensureSavedState();
228235
const storeState = Store.getState();
229236
const { tracing, dataset } = storeState;
@@ -263,6 +270,7 @@ export async function importTracingFiles(files: Array<File>, createGroupForEachF
263270
}
264271
}
265272

273+
await reader.close();
266274
return nmlImportActions;
267275
} catch (error) {
268276
// @ts-ignore

‎package.json

+2-3
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@
151151
"@types/pngjs": "^6.0.1",
152152
"@types/three": "^0.142.0",
153153
"@use-it/interval": "^1.0.0",
154+
"@zip.js/zip.js": "^2.6.81",
154155
"ansi-to-react": "^6.1.6",
155156
"antd": "^4.24.8",
156157
"backbone-events-standalone": "^0.2.7",
@@ -182,7 +183,6 @@
182183
"javascript-natural-sort": "^0.7.1",
183184
"js-priority-queue": "^0.1.5",
184185
"jsonschema": "^1.2.4",
185-
"jszip": "^3.7.0",
186186
"lodash": "^4.17.21",
187187
"lz-string": "^1.4.4",
188188
"lz4-wasm": "^0.9.2",
@@ -229,8 +229,7 @@
229229
"typed-redux-saga": "^1.4.0",
230230
"url": "^0.11.0",
231231
"url-join": "^4.0.0",
232-
"worker-loader": "^3.0.8",
233-
"zip-js-webpack": "^1.0.0"
232+
"worker-loader": "^3.0.8"
234233
},
235234
"resolutions": {
236235
"**/mini-store": "^1.1.0"

‎yarn.lock

+6-33
Original file line numberDiff line numberDiff line change
@@ -2311,6 +2311,11 @@
23112311
resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
23122312
integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
23132313

2314+
"@zip.js/zip.js@^2.6.81":
2315+
version "2.6.81"
2316+
resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.6.81.tgz#c3c9618a8e02f3a24d359a0a14d46985fea971f5"
2317+
integrity sha512-VXrwa5fthYq74sIZsHarCFVSwnKdispTd/WQBgcNEuB9X0N3L5s8odRCjx9Zw6XsvpG5krqB4ZN4X0lLMyjgDA==
2318+
23142319
JSONStream@^1.0.3, JSONStream@^1.0.4:
23152320
version "1.3.5"
23162321
resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
@@ -7168,11 +7173,6 @@ image-size@~0.5.0:
71687173
resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"
71697174
integrity sha1-Cd/Uq50g4p6xw+gLiZA3jfnjy5w=
71707175

7171-
immediate@~3.0.5:
7172-
version "3.0.6"
7173-
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
7174-
integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
7175-
71767176
immutability-helper@^3.1.1:
71777177
version "3.1.1"
71787178
resolved "https://registry.yarnpkg.com/immutability-helper/-/immutability-helper-3.1.1.tgz#2b86b2286ed3b1241c9e23b7b21e0444f52f77b7"
@@ -8161,16 +8161,6 @@ jsprim@^1.2.2:
81618161
json-schema "0.2.3"
81628162
verror "1.10.0"
81638163

8164-
jszip@^3.7.0:
8165-
version "3.7.0"
8166-
resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.7.0.tgz#9b8b995a4e7c9024653ce743e902076a82fdf4e6"
8167-
integrity sha512-Y2OlFIzrDOPWUnpU0LORIcDn2xN7rC9yKffFM/7pGhQuhO+SUhfm2trkJ/S5amjFvem0Y+1EALz/MEPkvHXVNw==
8168-
dependencies:
8169-
lie "~3.3.0"
8170-
pako "~1.0.2"
8171-
readable-stream "~2.3.6"
8172-
set-immediate-shim "~1.0.1"
8173-
81748164
just-extend@^4.0.2:
81758165
version "4.2.1"
81768166
resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
@@ -8313,13 +8303,6 @@ levn@~0.3.0:
83138303
prelude-ls "~1.1.2"
83148304
type-check "~0.3.2"
83158305

8316-
lie@~3.3.0:
8317-
version "3.3.0"
8318-
resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
8319-
integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
8320-
dependencies:
8321-
immediate "~3.0.5"
8322-
83238306
lines-and-columns@^1.1.6:
83248307
version "1.1.6"
83258308
resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -10027,7 +10010,7 @@ [email protected]:
1002710010
resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-1.0.0.tgz#9238e5480dedabacfe1fe3f2771063f164157d74"
1002810011
integrity sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==
1002910012

10030-
pako@^1.0.5, pako@~1.0.2:
10013+
pako@^1.0.5:
1003110014
version "1.0.11"
1003210015
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
1003310016
integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
@@ -12507,11 +12490,6 @@ set-blocking@^2.0.0:
1250712490
resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
1250812491
integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
1250912492

12510-
set-immediate-shim@~1.0.1:
12511-
version "1.0.1"
12512-
resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
12513-
integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=
12514-
1251512493
set-value@^2.0.0, set-value@^2.0.1:
1251612494
version "2.0.1"
1251712495
resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@@ -14672,11 +14650,6 @@ yocto-queue@^0.1.0:
1467214650
resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
1467314651
integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
1467414652

14675-
zip-js-webpack@^1.0.0:
14676-
version "1.0.0"
14677-
resolved "https://registry.yarnpkg.com/zip-js-webpack/-/zip-js-webpack-1.0.0.tgz#553dd15c76a9b2cfb4de97f338f4eb0fe0e2a897"
14678-
integrity sha512-epPHhnoh3nxrpEzM07yLZZtKANIubMeWUH2bslUIPMGo/vBDKJhdYqoupsfI6uT24i8tR9i7+y/ajC3Sa4R21A==
14679-
1468014653
zustand@^3.7.2:
1468114654
version "3.7.2"
1468214655
resolved "https://registry.yarnpkg.com/zustand/-/zustand-3.7.2.tgz#7b44c4f4a5bfd7a8296a3957b13e1c346f42514d"

0 commit comments

Comments
 (0)