From 6e521ff3a2fdb46bc0004e634bcef06cc9fb3b4e Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 30 Jan 2020 10:00:51 +0100 Subject: [PATCH] Produce TFLite-specific NPM package --- taskcluster/node-build.sh | 4 ++-- taskcluster/node-package-tflite.yml | 24 +++++++++++++++++++ taskcluster/scriptworker-task-github.yml | 3 +++ taskcluster/scriptworker-task-npm.yml | 3 +++ taskcluster/tc-tests-utils.sh | 23 +++++++++++------- ..._13x_16k_multiarchpkg-linux-tflite-opt.yml | 15 ++++++++++++ ...s_13x_8k_multiarchpkg-linux-tflite-opt.yml | 15 ++++++++++++ ...ejs_13x_multiarchpkg-darwin-tflite-opt.yml | 14 +++++++++++ ...nodejs_13x_multiarchpkg-win-tflite-opt.yml | 14 +++++++++++ 9 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 taskcluster/node-package-tflite.yml create mode 100644 taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml diff --git a/taskcluster/node-build.sh b/taskcluster/node-build.sh index 80a7e53ed6..8a89874036 100644 --- a/taskcluster/node-build.sh +++ b/taskcluster/node-build.sh @@ -2,10 +2,10 @@ set -xe -cuda=$1 +package_option=$1 source $(dirname "$0")/tc-tests-utils.sh source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh -do_deepspeech_npm_package "${cuda}" +do_deepspeech_npm_package "${package_option}" diff --git a/taskcluster/node-package-tflite.yml b/taskcluster/node-package-tflite.yml new file mode 100644 index 0000000000..0ab4022dc7 --- /dev/null +++ b/taskcluster/node-package-tflite.yml @@ -0,0 +1,24 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "darwin-amd64-tflite-opt" + - "linux-amd64-tflite-opt" + - "win-amd64-tflite-opt" + routes: + - "notify.irc-channel.${notifications.irc}.on-exception" + - "notify.irc-channel.${notifications.irc}.on-failed" + system_setup: + > + ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && + apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} + system_config: + > + ${swig.patch_nodejs.linux} + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + scripts: + build: "taskcluster/node-build.sh --tflite" + package: "taskcluster/node-package.sh" + metadata: + name: "DeepSpeech NodeJS TFLite package" + description: "Packaging DeepSpeech TFLite for registry" diff --git a/taskcluster/scriptworker-task-github.yml b/taskcluster/scriptworker-task-github.yml index 9729732c64..0460880a37 100644 --- a/taskcluster/scriptworker-task-github.yml +++ b/taskcluster/scriptworker-task-github.yml @@ -11,6 +11,7 @@ build: - "linux-rpi3-cpu-opt" - "node-package-gpu" - "node-package-cpu" + - "node-package-tflite" - "android-arm64-cpu-opt" - "android-armv7-cpu-opt" - "android-java-opt" @@ -41,6 +42,8 @@ build: - "node-package-gpu" # CPU package with all archs - "node-package-cpu" + # tflite package for non-default tflite archs + - "node-package-tflite" cpp: - "darwin-amd64-cpu-opt" - "linux-amd64-cpu-opt" diff --git a/taskcluster/scriptworker-task-npm.yml b/taskcluster/scriptworker-task-npm.yml index 0acf7b6eb1..1a4029634a 100644 --- a/taskcluster/scriptworker-task-npm.yml +++ b/taskcluster/scriptworker-task-npm.yml @@ -4,6 +4,7 @@ build: # Make sure builds are ready - "node-package-gpu" - "node-package-cpu" + - "node-package-tflite" allowed: - "tag" ref_match: "refs/tags/" @@ -21,6 +22,8 @@ build: - "node-package-gpu" # CPU package with all archs - "node-package-cpu" + # tflite package for non-default tflite archs + - "node-package-tflite" nuget: [] metadata: name: "DeepSpeech NPM Packages" diff --git a/taskcluster/tc-tests-utils.sh b/taskcluster/tc-tests-utils.sh index e2cf5f17bb..27974455b2 100755 --- a/taskcluster/tc-tests-utils.sh +++ b/taskcluster/tc-tests-utils.sh @@ -1192,14 +1192,17 @@ get_python_pkg_url() get_dep_npm_pkg_url() { local all_deps="$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${TASK_ID} | python -c 'import json; import sys; print(" ".join(json.loads(sys.stdin.read())["dependencies"]));')" - local deepspeech_pkg="deepspeech-${DS_VERSION}.tgz" - for dep in ${all_deps}; do - local has_artifact=$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts | python -c 'import json; import sys; has_artifact = True in [ e["name"].find("'${deepspeech_pkg}'") > 0 for e in json.loads(sys.stdin.read())["artifacts"] ]; print(has_artifact)') - if [ "${has_artifact}" = "True" ]; then - echo "https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts/public/${deepspeech_pkg}" - exit 0 - fi; + # We try "deepspeech-tflite" first and if we don't find it we try "deepspeech" + for pkg_basename in "deepspeech-tflite" "deepspeech"; do + local deepspeech_pkg="${pkg_basename}-${DS_VERSION}.tgz" + for dep in ${all_deps}; do + local has_artifact=$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts | python -c 'import json; import sys; has_artifact = True in [ e["name"].find("'${deepspeech_pkg}'") > 0 for e in json.loads(sys.stdin.read())["artifacts"] ]; print(has_artifact)') + if [ "${has_artifact}" = "True" ]; then + echo "https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts/public/${deepspeech_pkg}" + exit 0 + fi; + done; done; echo "" @@ -1435,7 +1438,7 @@ do_deepspeech_nodejs_build() do_deepspeech_npm_package() { - rename_to_gpu=$1 + package_option=$1 cd ${DS_DSDIR} @@ -1461,8 +1464,10 @@ do_deepspeech_npm_package() curl -L https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts/public/wrapper.tar.gz | tar -C native_client/javascript -xzvf - done; - if [ "${rename_to_gpu}" = "--cuda" ]; then + if [ "${package_option}" = "--cuda" ]; then make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-gpu + elif [ "${package_option}" = "--tflite" ]; then + make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-tflite else make -C native_client/javascript clean npm-pack fi diff --git a/taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml b/taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml new file mode 100644 index 0000000000..370aa323ef --- /dev/null +++ b/taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_13} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 13.x tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml b/taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml new file mode 100644 index 0000000000..0bfb723e67 --- /dev/null +++ b/taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-tflite" + - "test-training_8k-linux-amd64-py36m-opt" + test_model_task: "test-training_8k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_13} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 13.x tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml new file mode 100644 index 0000000000..fcda15936a --- /dev/null +++ b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" + metadata: + name: "DeepSpeech OSX AMD64 TFLite NodeJS MultiArch Package 13.x tests" + description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version" diff --git a/taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml b/taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml new file mode 100644 index 0000000000..7962d62d31 --- /dev/null +++ b/taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_13} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" + metadata: + name: "DeepSpeech Windows AMD64 TFLite NodeJS MultiArch Package 13.x tests" + description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version"