From 07df7a593c4838773df0dfe587ea6a0826f56db2 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 15 Dec 2021 15:55:48 +0100 Subject: [PATCH] Clippy fixes. (#846) * Clippy fixes. * Drop support for Python 3.6 * Remove other 3.6 * Re-enabling caches for build (5h + seems too long and issue seems solved) https://github.com/actions/virtual-environments/issues/572 * `npm audit fix`. * Fix yaml ? * Pyarrow issue fixed: https://github.com/huggingface/datasets/pull/2268 * Installing dev libraries. * Install python dev elsewhere ? * Typo. * No sudo. * ... * Testing the GH again. * Maybe v2 will fix ? * Fixing tests on MacOS Python 3.8+ --- .github/workflows/docs-check.yml | 2 +- .github/workflows/docs-deploy.yml | 2 +- .github/workflows/python-release-conda.yml | 2 +- .github/workflows/python-release.yml | 4 +- .github/workflows/python.yml | 36 +- bindings/node/native/src/models.rs | 16 +- bindings/node/native/src/normalizers.rs | 2 +- bindings/node/native/src/tasks/tokenizer.rs | 6 +- bindings/node/native/src/tokenizer.rs | 4 +- bindings/node/package-lock.json | 637 ++++++++++---------- bindings/node/package.json | 2 +- bindings/python/src/encoding.rs | 4 +- bindings/python/src/models.rs | 4 +- bindings/python/src/pre_tokenizers.rs | 2 +- bindings/python/src/processors.rs | 4 +- bindings/python/src/utils/normalization.rs | 4 +- bindings/python/tests/utils.py | 4 + tokenizers/src/models/unigram/lattice.rs | 8 +- tokenizers/src/normalizers/utils.rs | 4 +- tokenizers/src/pre_tokenizers/byte_level.rs | 8 +- tokenizers/src/processors/roberta.rs | 8 +- tokenizers/tests/unigram.rs | 2 +- 22 files changed, 364 insertions(+), 401 deletions(-) diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index ec42cf312..1791f9240 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -16,7 +16,7 @@ jobs: - name: Install Python uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.9 - name: Install dependencies run: pip install sphinx sphinx_rtd_theme setuptools-rust diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index ec3e92afe..2ecbbf6d2 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install Python uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.9 - name: Install Python dependencies run: pip install sphinx sphinx_rtd_theme setuptools-rust diff --git a/.github/workflows/python-release-conda.yml b/.github/workflows/python-release-conda.yml index 4019835b8..a604bc9bf 100644 --- a/.github/workflows/python-release-conda.yml +++ b/.github/workflows/python-release-conda.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python: [3.6, 3.7, 3.8, 3.9] + python: [3.7, 3.8, 3.9] defaults: run: shell: bash -l {0} diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index cb6201059..9ce708afe 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -30,7 +30,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9] + python: [3.7, 3.8, 3.9] steps: - name: Checkout repository uses: actions/checkout@v1 @@ -76,7 +76,7 @@ jobs: strategy: matrix: os: [windows-latest, macos-10.15] - python: [3.6, 3.7, 3.8, 3.9] + python: [3.7, 3.8, 3.9] steps: - name: Checkout repository uses: actions/checkout@v1 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 92d96c02d..49ff9af5a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -16,7 +16,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9] + python: [3.7, 3.8, 3.9] steps: - name: Checkout repository uses: actions/checkout@v1 @@ -34,7 +34,7 @@ jobs: - name: Install Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} architecture: x86 @@ -45,6 +45,7 @@ jobs: command: build args: --manifest-path ./bindings/python/Cargo.toml + build_and_test: name: Check everything builds & tests runs-on: ${{ matrix.os }} @@ -54,6 +55,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v1 + - name: Install Rust uses: actions-rs/toolchain@v1 @@ -61,20 +63,17 @@ jobs: toolchain: stable components: rustfmt, clippy - # # Necessary for now for the cargo cache: https://github.com/actions/cache/issues/133#issuecomment-599102035 - # - run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - # - name: Cache Cargo Registry - # uses: actions/cache@v1 - # with: - # path: ~/.cargo/registry - # key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} + - name: Cache Cargo Registry + uses: actions/cache@v1 + with: + path: ~/.cargo/registry + key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - # - name: Cache Cargo Build Target - # uses: actions/cache@v1 - # with: - # path: ./bindings/python/target - # key: ${{ runner.os }}-cargo-python-build-${{ hashFiles('**/Cargo.lock') }} + - name: Cache Cargo Build Target + uses: actions/cache@v1 + with: + path: ./bindings/python/target + key: ${{ runner.os }}-cargo-python-build-${{ hashFiles('**/Cargo.lock') }} - name: Lint with RustFmt uses: actions-rs/cargo@v1 @@ -90,9 +89,9 @@ jobs: args: --manifest-path ./bindings/python/Cargo.toml --all-targets --all-features -- -D warnings - name: Install Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.6 + python-version: 3.9 architecture: "x64" - name: Install @@ -100,7 +99,8 @@ jobs: run: | python -m venv .env source .env/bin/activate - pip install pytest requests setuptools_rust numpy "pyarrow<3.0.0" datasets + pip install -U pip + pip install pytest requests setuptools_rust numpy pyarrow datasets python setup.py develop - name: Check style diff --git a/bindings/node/native/src/models.rs b/bindings/node/native/src/models.rs index a64187e7b..89cf98d22 100644 --- a/bindings/node/native/src/models.rs +++ b/bindings/node/native/src/models.rs @@ -168,9 +168,7 @@ impl BpeOptions { fn bpe_init(mut cx: FunctionContext) -> JsResult { let vocab = cx.extract::(0)?; let merges = cx.extract::(1)?; - let options = cx - .extract_opt::(2)? - .unwrap_or_else(BpeOptions::default); + let options = cx.extract_opt::(2)?.unwrap_or_default(); let mut builder = tk::models::bpe::BPE::builder().vocab_and_merges(vocab, merges); builder = options.apply_to_bpe_builder(builder); @@ -251,9 +249,7 @@ impl WordPieceOptions { /// }) fn wordpiece_init(mut cx: FunctionContext) -> JsResult { let vocab = cx.extract::>(0)?; - let options = cx - .extract_opt::(1)? - .unwrap_or_else(WordPieceOptions::default); + let options = cx.extract_opt::(1)?.unwrap_or_default(); let mut builder = tk::models::wordpiece::WordPiece::builder().vocab(vocab); builder = options.apply_to_wordpiece_builder(builder); @@ -320,9 +316,7 @@ impl WordLevelOptions { /// }, callback) fn wordlevel_init(mut cx: FunctionContext) -> JsResult { let vocab = cx.extract::>(0)?; - let options = cx - .extract_opt::(1)? - .unwrap_or_else(WordLevelOptions::default); + let options = cx.extract_opt::(1)?.unwrap_or_default(); let mut builder = tk::models::wordlevel::WordLevel::builder().vocab(vocab); builder = options.apply_to_wordlevel_builder(builder); @@ -378,9 +372,7 @@ struct UnigramOptions { /// }) fn unigram_init(mut cx: FunctionContext) -> JsResult { let vocab = cx.extract::>(0)?; - let options = cx - .extract_opt::(1)? - .unwrap_or_else(UnigramOptions::default); + let options = cx.extract_opt::(1)?.unwrap_or_default(); let unigram = tk::models::unigram::Unigram::from(vocab, options.unk_id) .map_err(|e| Error(e.to_string()))?; diff --git a/bindings/node/native/src/normalizers.rs b/bindings/node/native/src/normalizers.rs index b2ec51dce..0e5fa0c0b 100644 --- a/bindings/node/native/src/normalizers.rs +++ b/bindings/node/native/src/normalizers.rs @@ -115,7 +115,7 @@ impl Default for BertNormalizerOptions { fn bert_normalizer(mut cx: FunctionContext) -> JsResult { let options = cx .extract_opt::(0)? - .unwrap_or_else(BertNormalizerOptions::default); + .unwrap_or_default(); let mut normalizer = JsNormalizer::new::<_, JsNormalizer, _>(&mut cx, vec![])?; let guard = cx.lock(); diff --git a/bindings/node/native/src/tasks/tokenizer.rs b/bindings/node/native/src/tasks/tokenizer.rs index 7e96d7a0a..2a7e3e0ad 100644 --- a/bindings/node/native/src/tasks/tokenizer.rs +++ b/bindings/node/native/src/tasks/tokenizer.rs @@ -11,7 +11,7 @@ pub enum EncodeTask<'s> { } pub enum EncodeOutput { - Single(Encoding), + Single(Box), Batch(Vec), } @@ -35,7 +35,7 @@ impl Task for EncodeTask<'static> { *add_special_tokens, ) .map_err(|e| format!("{}", e)) - .map(EncodeOutput::Single) + .map(|item| EncodeOutput::Single(Box::new(item))) } EncodeTask::Batch(worker, input, add_special_tokens) => { let mut input: Option> = @@ -65,7 +65,7 @@ impl Task for EncodeTask<'static> { let mut js_encoding = JsEncoding::new::<_, JsEncoding, _>(&mut cx, vec![])?; // Set the actual encoding let guard = cx.lock(); - js_encoding.borrow_mut(&guard).encoding = Some(encoding); + js_encoding.borrow_mut(&guard).encoding = Some(*encoding); Ok(js_encoding.upcast()) } EncodeOutput::Batch(encodings) => { diff --git a/bindings/node/native/src/tokenizer.rs b/bindings/node/native/src/tokenizer.rs index e8058aa65..2b705ed1b 100644 --- a/bindings/node/native/src/tokenizer.rs +++ b/bindings/node/native/src/tokenizer.rs @@ -71,7 +71,7 @@ declare_types! { let content = cx.extract::(0)?; let special = cx.extract::(1)?; let token = cx.extract_opt::(2)? - .unwrap_or_else(AddedTokenOptions::default) + .unwrap_or_default() .into_added_token(content, special); Ok(AddedToken { token }) @@ -1034,7 +1034,7 @@ pub fn tokenizer_from_pretrained(mut cx: FunctionContext) -> JsResult(0)?; let mut p: tk::FromPretrainedParameters = cx .extract_opt::(1)? - .unwrap_or_else(FromPretrainedParametersJs::default) + .unwrap_or_default() .into(); p.user_agent = [("bindings", "Node.js"), ("version", crate::VERSION)] diff --git a/bindings/node/package-lock.json b/bindings/node/package-lock.json index 6f4beb1a5..ba850b1f5 100644 --- a/bindings/node/package-lock.json +++ b/bindings/node/package-lock.json @@ -628,9 +628,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "rimraf": { @@ -1141,23 +1141,6 @@ "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", "dev": true }, - "ansi-escape-sequences": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-escape-sequences/-/ansi-escape-sequences-4.1.0.tgz", - "integrity": "sha512-dzW9kHxH011uBsidTXd14JXgzye/YLb2LzeKZ4bsgl/Knwx8AtbSFkkGxagdNOoh0DlqHCmfiEjWKBaqjOanVw==", - "dev": true, - "requires": { - "array-back": "^3.0.1" - }, - "dependencies": { - "array-back": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", - "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", - "dev": true - } - } - }, "ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -1241,13 +1224,10 @@ "dev": true }, "array-back": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-2.0.0.tgz", - "integrity": "sha512-eJv4pLLufP3g5kcZry0j6WXpIbzYw9GUB4mVJZno9wfwiBxbizTnHCw3VJb07cBihbFX48Y7oSrW9y+gt4glyw==", - "dev": true, - "requires": { - "typical": "^2.6.1" - } + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "dev": true }, "array-unique": { "version": "0.3.2", @@ -1563,6 +1543,12 @@ "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -1603,6 +1589,21 @@ } } }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "dev": true + }, "cliui": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", @@ -1615,9 +1616,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "is-fullwidth-code-point": { @@ -1706,35 +1707,84 @@ } }, "command-line-args": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-4.0.7.tgz", - "integrity": "sha512-aUdPvQRAyBvQd2n7jXcsMDz68ckBJELXNzBybCHOibUWEg0mWTnaYCSRU8h9R+aNRSvDihJtssSRCiDRpLaezA==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.0.tgz", + "integrity": "sha512-4zqtU1hYsSJzcJBOcNZIbW5Fbk9BkjCp1pZVhQKoRaWL5J7N4XphDLwo8aWwdQpTugxwu+jf9u2ZhkXiqp5Z6A==", "dev": true, "requires": { - "array-back": "^2.0.0", - "find-replace": "^1.0.3", - "typical": "^2.6.1" + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" } }, "command-line-commands": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/command-line-commands/-/command-line-commands-2.0.1.tgz", - "integrity": "sha512-m8c2p1DrNd2ruIAggxd/y6DgygQayf6r8RHwchhXryaLF8I6koYjoYroVP+emeROE9DXN5b9sP1Gh+WtvTTdtQ==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/command-line-commands/-/command-line-commands-3.0.2.tgz", + "integrity": "sha512-ac6PdCtdR6q7S3HN+JiVLIWGHY30PRYIEl2qPo+FuEuzwAUk0UYyimrngrg7FvF/mCr4Jgoqv5ZnHZgads50rw==", "dev": true, "requires": { - "array-back": "^2.0.0" + "array-back": "^4.0.1" + }, + "dependencies": { + "array-back": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", + "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "dev": true + } } }, "command-line-usage": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-4.1.0.tgz", - "integrity": "sha512-MxS8Ad995KpdAC0Jopo/ovGIroV/m0KHwzKfXxKag6FHOkGsH8/lv5yjgablcRxCJJC0oJeUMuO/gmaq+Wq46g==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.1.tgz", + "integrity": "sha512-F59pEuAR9o1SF/bD0dQBDluhpT4jJQNWUHEuVBqpDmCUo6gPjCi+m9fCWnWZVR/oG6cMTUms4h+3NPl74wGXvA==", "dev": true, "requires": { - "ansi-escape-sequences": "^4.0.0", - "array-back": "^2.0.0", - "table-layout": "^0.4.2", - "typical": "^2.6.1" + "array-back": "^4.0.1", + "chalk": "^2.4.2", + "table-layout": "^1.0.1", + "typical": "^5.2.0" + }, + "dependencies": { + "array-back": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", + "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "typical": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", + "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "dev": true + } } }, "comment-parser": { @@ -2128,9 +2178,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "debug": { @@ -2566,6 +2616,17 @@ } } }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, "extglob": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", @@ -2664,6 +2725,15 @@ "bser": "2.1.1" } }, + "figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -2683,24 +2753,12 @@ } }, "find-replace": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-1.0.3.tgz", - "integrity": "sha1-uI5zZNLZyVlVnziMZmcNYTBEH6A=", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", "dev": true, "requires": { - "array-back": "^1.0.4", - "test-value": "^2.1.0" - }, - "dependencies": { - "array-back": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", - "integrity": "sha1-ZEun8JX3/898Q7Xw3DnTwfA8Bjs=", - "dev": true, - "requires": { - "typical": "^2.6.0" - } - } + "array-back": "^3.0.1" } }, "find-up": { @@ -3133,6 +3191,61 @@ "integrity": "sha1-g21r7+bfv87gvM8c+fKsxwJ/eD0=", "dev": true }, + "inquirer": { + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.3.3.tgz", + "integrity": "sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==", + "dev": true, + "requires": { + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.19", + "mute-stream": "0.0.8", + "run-async": "^2.4.0", + "rxjs": "^6.6.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, "interpret": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", @@ -4053,6 +4166,12 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, + "lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", + "dev": true + }, "lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -4065,12 +4184,6 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "lodash.padend": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz", - "integrity": "sha1-U8y6BH0G4VjTEfRdpiX05J5vFm4=", - "dev": true - }, "lodash.truncate": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", @@ -4117,6 +4230,12 @@ "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, + "make-promises-safe": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/make-promises-safe/-/make-promises-safe-5.1.0.tgz", + "integrity": "sha512-AfdZ49rtyhQR/6cqVKGoH7y4ql7XkS5HJI1lZm0/5N6CQosy1eYbBJ/qbhkKHzo17UH7M918Bysf6XB9f3kS1g==", + "dev": true + }, "makeerror": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.11.tgz", @@ -4242,6 +4361,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -4284,184 +4409,43 @@ "dev": true }, "neon-cli": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/neon-cli/-/neon-cli-0.3.3.tgz", - "integrity": "sha512-P2kc9ZDp0fXfmHDx3YcQwX8gvZew8WroXG3YY6N2LZoYZNrnGUWIOYGQcck52Jmry+ka02xDtw+lCHpOuGoxRA==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/neon-cli/-/neon-cli-0.9.1.tgz", + "integrity": "sha512-iWwKjI1Q4o1lWVc4vkr59BXEYlBl5HggKx5ETw5kStpuvT1nVcolPtTdydisx8pLRRxN+gjER571DOGnA0Yxcg==", "dev": true, "requires": { - "chalk": "~2.1.0", - "command-line-args": "^4.0.2", - "command-line-commands": "^2.0.0", - "command-line-usage": "^4.0.0", + "chalk": "^4.1.0", + "command-line-args": "^5.1.1", + "command-line-commands": "^3.0.1", + "command-line-usage": "^6.1.0", "git-config": "0.0.7", - "handlebars": "^4.1.0", - "inquirer": "^3.0.6", - "mkdirp": "^0.5.1", - "quickly-copy-file": "^1.0.0", - "rimraf": "^2.6.1", - "rsvp": "^4.6.1", - "semver": "^5.1.0", - "toml": "^2.3.0", - "ts-typed-json": "^0.2.2", - "validate-npm-package-license": "^3.0.1", + "handlebars": "^4.7.6", + "inquirer": "^7.3.3", + "make-promises-safe": "^5.1.0", + "rimraf": "^3.0.2", + "semver": "^7.3.2", + "toml": "^3.0.0", + "ts-typed-json": "^0.3.2", + "validate-npm-package-license": "^3.0.4", "validate-npm-package-name": "^3.0.0" }, "dependencies": { - "ansi-escapes": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", - "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", - "dev": true - }, - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true - }, - "chalk": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.1.0.tgz", - "integrity": "sha512-LUHGS/dge4ujbXMJrnihYMcL4AoOweGnw9Tp3kQuqy1Kx5c1qKjqvMJZ6nVJPMWJtKCTN72ZogH3oeSO9g9rXQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.1.0", - "escape-string-regexp": "^1.0.5", - "supports-color": "^4.0.0" - } - }, - "chardet": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", - "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=", - "dev": true - }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", - "dev": true, - "requires": { - "restore-cursor": "^2.0.0" - } - }, - "cli-width": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", - "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", - "dev": true - }, - "external-editor": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz", - "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==", - "dev": true, - "requires": { - "chardet": "^0.4.0", - "iconv-lite": "^0.4.17", - "tmp": "^0.0.33" - } - }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", - "dev": true, - "requires": { - "escape-string-regexp": "^1.0.5" - } - }, - "has-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", - "dev": true - }, - "inquirer": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz", - "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==", - "dev": true, - "requires": { - "ansi-escapes": "^3.0.0", - "chalk": "^2.0.0", - "cli-cursor": "^2.1.0", - "cli-width": "^2.0.0", - "external-editor": "^2.0.4", - "figures": "^2.0.0", - "lodash": "^4.3.0", - "mute-stream": "0.0.7", - "run-async": "^2.2.0", - "rx-lite": "^4.0.8", - "rx-lite-aggregates": "^4.0.8", - "string-width": "^2.1.0", - "strip-ansi": "^4.0.0", - "through": "^2.3.6" - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true - }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, - "mute-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", - "dev": true - }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", - "dev": true, - "requires": { - "mimic-fn": "^1.0.0" - } - }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", - "dev": true, - "requires": { - "onetime": "^2.0.0", - "signal-exit": "^3.0.2" - } - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "dev": true, "requires": { - "ansi-regex": "^3.0.0" + "glob": "^7.1.3" } }, - "supports-color": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz", - "integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=", + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", "dev": true, "requires": { - "has-flag": "^2.0.0" + "lru-cache": "^6.0.0" } } } @@ -4883,9 +4867,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "ansi-styles": { @@ -4957,15 +4941,6 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, - "quickly-copy-file": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/quickly-copy-file/-/quickly-copy-file-1.0.0.tgz", - "integrity": "sha1-n4/wZiMFEO50IrASFHKwk6hpCFk=", - "dev": true, - "requires": { - "mkdirp": "~0.5.0" - } - }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -5046,9 +5021,9 @@ } }, "reduce-flatten": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-1.0.1.tgz", - "integrity": "sha1-JYx479FT3fk8tWEjf2EYTzaW4yc=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", + "integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==", "dev": true }, "regex-not": { @@ -5147,6 +5122,16 @@ "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=", "dev": true }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", @@ -5173,19 +5158,13 @@ "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", "dev": true }, - "rx-lite": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz", - "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=", - "dev": true - }, - "rx-lite-aggregates": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz", - "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=", + "rxjs": { + "version": "6.6.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", + "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", "dev": true, "requires": { - "rx-lite": "*" + "tslib": "^1.9.0" } }, "safe-buffer": { @@ -5747,9 +5726,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "strip-ansi": { @@ -5864,9 +5843,9 @@ } }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "is-fullwidth-code-point": { @@ -5904,16 +5883,29 @@ } }, "table-layout": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-0.4.5.tgz", - "integrity": "sha512-zTvf0mcggrGeTe/2jJ6ECkJHAQPIYEwDoqsiqBjI24mvRmQbInK5jq33fyypaCBxX08hMkfmdOqj6haT33EqWw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz", + "integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==", "dev": true, "requires": { - "array-back": "^2.0.0", + "array-back": "^4.0.1", "deep-extend": "~0.6.0", - "lodash.padend": "^4.6.1", - "typical": "^2.6.1", - "wordwrapjs": "^3.0.0" + "typical": "^5.2.0", + "wordwrapjs": "^4.0.0" + }, + "dependencies": { + "array-back": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz", + "integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==", + "dev": true + }, + "typical": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", + "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "dev": true + } } }, "tar": { @@ -5958,27 +5950,6 @@ "minimatch": "^3.0.4" } }, - "test-value": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/test-value/-/test-value-2.1.0.tgz", - "integrity": "sha1-Edpv9nDzRxpztiXKTz/c97t0gpE=", - "dev": true, - "requires": { - "array-back": "^1.0.3", - "typical": "^2.6.0" - }, - "dependencies": { - "array-back": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/array-back/-/array-back-1.0.4.tgz", - "integrity": "sha1-ZEun8JX3/898Q7Xw3DnTwfA8Bjs=", - "dev": true, - "requires": { - "typical": "^2.6.0" - } - } - } - }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -6007,9 +5978,9 @@ } }, "tmpl": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", - "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, "to-fast-properties": { @@ -6060,9 +6031,9 @@ } }, "toml": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/toml/-/toml-2.3.6.tgz", - "integrity": "sha512-gVweAectJU3ebq//Ferr2JUY4WKSDe5N+z0FvjDncLGyHmIDoxgY/2Ie4qfEIDm4IS7OA6Rmdm7pdEEdMcV/xQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==", "dev": true }, "tough-cookie": { @@ -6127,21 +6098,10 @@ } }, "ts-typed-json": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/ts-typed-json/-/ts-typed-json-0.2.2.tgz", - "integrity": "sha1-UxhL7ok+RZkbc8jEY6OLWeJ81H4=", - "dev": true, - "requires": { - "rsvp": "^3.5.0" - }, - "dependencies": { - "rsvp": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-3.6.2.tgz", - "integrity": "sha512-OfWGQTb9vnwRjwtA2QwpG2ICclHC3pgXZO5xt8H2EfgDquO0qVdSb5T88L4qJVAEugbS56pAuV4XZM58UX8ulw==", - "dev": true - } - } + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/ts-typed-json/-/ts-typed-json-0.3.2.tgz", + "integrity": "sha512-Tdu3BWzaer7R5RvBIJcg9r8HrTZgpJmsX+1meXMJzYypbkj8NK2oJN0yvm4Dp/Iv6tzFa/L5jKRmEVTga6K3nA==", + "dev": true }, "tslib": { "version": "1.14.1", @@ -6195,15 +6155,15 @@ "dev": true }, "typical": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", - "integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", "dev": true }, "uglify-js": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.10.0.tgz", - "integrity": "sha512-Esj5HG5WAyrLIdYU74Z3JdG2PxdIusvj6IWHMtlyESxc7kcDz7zYlYjpnSokn1UbpV0d/QX9fan7gkCNd/9BQA==", + "version": "3.14.5", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.14.5.tgz", + "integrity": "sha512-qZukoSxOG0urUTvjc2ERMTcAy+BiFh3weWAkeurLwjrCba73poHmG3E36XEjd/JGukMzwTL7uCxZiAexj8ppvQ==", "dev": true, "optional": true }, @@ -6437,13 +6397,21 @@ "dev": true }, "wordwrapjs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-3.0.0.tgz", - "integrity": "sha512-mO8XtqyPvykVCsrwj5MlOVWvSnCdT+C+QVbm6blradR7JExAhbkZ7hZ9A+9NUtwzSqrlUo9a67ws0EiILrvRpw==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz", + "integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==", "dev": true, "requires": { - "reduce-flatten": "^1.0.1", - "typical": "^2.6.1" + "reduce-flatten": "^2.0.0", + "typical": "^5.2.0" + }, + "dependencies": { + "typical": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz", + "integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==", + "dev": true + } } }, "wrap-ansi": { @@ -6458,9 +6426,9 @@ }, "dependencies": { "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "ansi-styles": { @@ -6582,8 +6550,7 @@ "dependencies": { "ansi-regex": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "resolved": "", "dev": true }, "is-fullwidth-code-point": { diff --git a/bindings/node/package.json b/bindings/node/package.json index 3642abf28..fb5e0ac17 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -29,7 +29,7 @@ "eslint-plugin-prettier": "^3.4.1", "eslint-plugin-simple-import-sort": "^5.0.3", "jest": "^26.6.3", - "neon-cli": "^0.3.3", + "neon-cli": "^0.9.1", "prettier": "^2.3.2", "shelljs": "^0.8.3", "ts-jest": "^26.5.6", diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs index a88a46006..fbc6bfe0a 100644 --- a/bindings/python/src/encoding.rs +++ b/bindings/python/src/encoding.rs @@ -51,7 +51,7 @@ impl PyEncoding { let data = serde_json::to_string(&self.encoding).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to pickle Encoding: {}", - e.to_string() + e )) })?; Ok(PyBytes::new(py, data.as_bytes()).to_object(py)) @@ -63,7 +63,7 @@ impl PyEncoding { self.encoding = serde_json::from_slice(s.as_bytes()).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to unpickle Encoding: {}", - e.to_string() + e )) })?; Ok(()) diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index 1c0e84195..e7aaa79bc 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -103,7 +103,7 @@ impl PyModel { let data = serde_json::to_string(&self.model).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to pickle Model: {}", - e.to_string() + e )) })?; Ok(PyBytes::new(py, data.as_bytes()).to_object(py)) @@ -115,7 +115,7 @@ impl PyModel { self.model = serde_json::from_slice(s.as_bytes()).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to unpickle Model: {}", - e.to_string() + e )) })?; Ok(()) diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index 77840012b..947e267e0 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -112,7 +112,7 @@ impl PyPreTokenizer { let data = serde_json::to_string(&self.pretok).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to pickle PreTokenizer: {}", - e.to_string() + e )) })?; Ok(PyBytes::new(py, data.as_bytes()).to_object(py)) diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs index 757ae4010..12990b3d4 100644 --- a/bindings/python/src/processors.rs +++ b/bindings/python/src/processors.rs @@ -71,7 +71,7 @@ impl PyPostProcessor { let data = serde_json::to_string(self.processor.as_ref()).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to pickle PostProcessor: {}", - e.to_string() + e )) })?; Ok(PyBytes::new(py, data.as_bytes()).to_object(py)) @@ -83,7 +83,7 @@ impl PyPostProcessor { self.processor = serde_json::from_slice(s.as_bytes()).map_err(|e| { exceptions::PyException::new_err(format!( "Error while attempting to unpickle PostProcessor: {}", - e.to_string() + e )) })?; Ok(()) diff --git a/bindings/python/src/utils/normalization.rs b/bindings/python/src/utils/normalization.rs index 59380b9de..219a6659c 100644 --- a/bindings/python/src/utils/normalization.rs +++ b/bindings/python/src/utils/normalization.rs @@ -559,7 +559,7 @@ impl PyNormalizedStringRefMut { fn filter(&mut self, func: &PyAny) -> PyResult<()> { self.inner - .map_mut(|mut n| filter(&mut n, func)) + .map_mut(|n| filter(n, func)) .ok_or_else(PyNormalizedStringRefMut::destroyed_error)??; Ok(()) } @@ -573,7 +573,7 @@ impl PyNormalizedStringRefMut { fn map(&mut self, func: &PyAny) -> PyResult<()> { self.inner - .map_mut(|mut n| map(&mut n, func)) + .map_mut(|n| map(n, func)) .ok_or_else(PyNormalizedStringRefMut::destroyed_error)??; Ok(()) } diff --git a/bindings/python/tests/utils.py b/bindings/python/tests/utils.py index 02bfb5b24..b2b028dba 100644 --- a/bindings/python/tests/utils.py +++ b/bindings/python/tests/utils.py @@ -98,6 +98,10 @@ def doc_pipeline_bert_tokenizer(data_dir): ) +# On MacOS Python 3.8+ the default was modified to `spawn`, we need `fork` in tests. +mp.set_start_method("fork") + + def multiprocessing_with_parallelism(tokenizer, enabled: bool): """ This helper can be used to test that disabling parallelism avoids dead locks when the diff --git a/tokenizers/src/models/unigram/lattice.rs b/tokenizers/src/models/unigram/lattice.rs index 9a620c030..897d16844 100644 --- a/tokenizers/src/models/unigram/lattice.rs +++ b/tokenizers/src/models/unigram/lattice.rs @@ -56,8 +56,8 @@ pub struct Lattice<'a> { nodes: Vec, pub(super) begin_nodes: Vec>, pub(super) end_nodes: Vec>, - bos_id: usize, - eos_id: usize, + _bos_id: usize, + _eos_id: usize, } impl std::fmt::Display for Lattice<'_> { @@ -158,8 +158,8 @@ impl<'a> Lattice<'a> { nodes, begin_nodes, end_nodes, - bos_id, - eos_id, + _bos_id: bos_id, + _eos_id: eos_id, } } diff --git a/tokenizers/src/normalizers/utils.rs b/tokenizers/src/normalizers/utils.rs index cdccb190a..6906e5768 100644 --- a/tokenizers/src/normalizers/utils.rs +++ b/tokenizers/src/normalizers/utils.rs @@ -26,9 +26,9 @@ impl Sequence { } impl Normalizer for Sequence { - fn normalize(&self, mut normalized: &mut NormalizedString) -> Result<()> { + fn normalize(&self, normalized: &mut NormalizedString) -> Result<()> { for normalizer in &self.normalizers { - normalizer.normalize(&mut normalized)?; + normalizer.normalize(normalized)?; } Ok(()) } diff --git a/tokenizers/src/pre_tokenizers/byte_level.rs b/tokenizers/src/pre_tokenizers/byte_level.rs index 794f0d8aa..0bb5eee88 100644 --- a/tokenizers/src/pre_tokenizers/byte_level.rs +++ b/tokenizers/src/pre_tokenizers/byte_level.rs @@ -156,14 +156,14 @@ impl PostProcessor for ByteLevel { encoding .get_overflowing_mut() .iter_mut() - .for_each(|mut encoding| process_offsets(&mut encoding, self.add_prefix_space)); + .for_each(|encoding| process_offsets(encoding, self.add_prefix_space)); - if let Some(mut encoding) = pair_encoding.as_mut() { - process_offsets(&mut encoding, self.add_prefix_space); + if let Some(encoding) = pair_encoding.as_mut() { + process_offsets(encoding, self.add_prefix_space); encoding .get_overflowing_mut() .iter_mut() - .for_each(|mut encoding| process_offsets(&mut encoding, self.add_prefix_space)); + .for_each(|encoding| process_offsets(encoding, self.add_prefix_space)); } } diff --git a/tokenizers/src/processors/roberta.rs b/tokenizers/src/processors/roberta.rs index 0036a7e89..cbdd03eb7 100644 --- a/tokenizers/src/processors/roberta.rs +++ b/tokenizers/src/processors/roberta.rs @@ -62,14 +62,14 @@ impl PostProcessor for RobertaProcessing { encoding .get_overflowing_mut() .iter_mut() - .for_each(|mut encoding| process_offsets(&mut encoding, self.add_prefix_space)); + .for_each(|encoding| process_offsets(encoding, self.add_prefix_space)); - if let Some(mut encoding) = pair_encoding.as_mut() { - process_offsets(&mut encoding, self.add_prefix_space); + if let Some(encoding) = pair_encoding.as_mut() { + process_offsets(encoding, self.add_prefix_space); encoding .get_overflowing_mut() .iter_mut() - .for_each(|mut encoding| process_offsets(&mut encoding, self.add_prefix_space)); + .for_each(|encoding| process_offsets(encoding, self.add_prefix_space)); } } diff --git a/tokenizers/tests/unigram.rs b/tokenizers/tests/unigram.rs index fd24782c6..b0d906797 100644 --- a/tokenizers/tests/unigram.rs +++ b/tokenizers/tests/unigram.rs @@ -44,7 +44,7 @@ fn test_train_unigram_from_file() { let mut word_counts = HashMap::new(); content.split_whitespace().for_each(|word| { // This is important for the test of char vs u8 - let word = format!("▁{}", word.to_string()); + let word = format!("▁{}", word); *word_counts.entry(word).or_insert(0) += 1; });