From d8cb2a75e057cc27a83f7d4029f8e94f829069e3 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Fri, 12 Apr 2019 16:34:08 -0400 Subject: [PATCH 1/6] Initial working example for bert qa --- .../examples/bert-qa/.gitignore | 12 ++ .../examples/bert-qa/README.md | 73 +++++++++ .../examples/bert-qa/get_bert_data.sh | 29 ++++ .../examples/bert-qa/project.clj | 10 ++ .../examples/bert-qa/squad-samples.edn | 17 +++ .../examples/bert-qa/src/bert_qa/core.clj | 144 ++++++++++++++++++ .../bert-qa/test/bert_qa/core_test.clj | 7 + 7 files changed, 292 insertions(+) create mode 100644 contrib/clojure-package/examples/bert-qa/.gitignore create mode 100644 contrib/clojure-package/examples/bert-qa/README.md create mode 100755 contrib/clojure-package/examples/bert-qa/get_bert_data.sh create mode 100644 contrib/clojure-package/examples/bert-qa/project.clj create mode 100644 contrib/clojure-package/examples/bert-qa/squad-samples.edn create mode 100644 contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj create mode 100644 contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj diff --git a/contrib/clojure-package/examples/bert-qa/.gitignore b/contrib/clojure-package/examples/bert-qa/.gitignore new file mode 100644 index 000000000000..d18f225992a9 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/.gitignore @@ -0,0 +1,12 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +.hgignore +.hg/ diff --git a/contrib/clojure-package/examples/bert-qa/README.md b/contrib/clojure-package/examples/bert-qa/README.md new file mode 100644 index 000000000000..a61e2709d145 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/README.md @@ -0,0 +1,73 @@ +# bert-qa + +**This example was based off of the Java API one. It shows how to do inference with a pre-trained BERT network that is trained on Questions and Answers using the [SQuAD Dataset](https://rajpurkar.github.io/SQuAD-explorer/)** + +The pretrained model was created using GluonNLP and then exported to the MXNet symbol format. You can find more information in the background section below. + +In this tutorial, we will walk through the BERT QA model trained by MXNet. +Users can provide a question with a paragraph contains answer to the model and +the model will be able to find the best answer from the answer paragraph. + +Example: + +``` +{:input-answer "Steam engines are external combustion engines, where the working fluid is separate from the combustion products. Non-combustion heat sources such as solar power, nuclear power or geothermal energy may be used. The ideal thermodynamic cycle used to analyze this process is called the Rankine cycle. In the cycle, water is heated and transforms into steam within a boiler operating at a high pressure. When expanded through pistons or turbines, mechanical work is done. The reduced-pressure steam is then condensed and pumped back into the boiler." + :input-question "Along with geothermal and nuclear, what is a notable non-combustion heat source?" + :ground-truth-answers ["solar" + "solar power" + "solar power, nuclear power or geothermal energysolar"]} +``` + +The prediction in this case would be `solar power` + +## Setup Guide + +### Step 1: Download the model + +For this tutorial, you can get the model and vocabulary by running following bash file. This script will use `wget` to download these artifacts from AWS S3. + +From the `scala-package/examples/scripts/infer/bert/` folder run: + +```bash +./get_bert_data.sh +``` + +Some sample questions and answers are provide in the `squad-sample.edn` file. Some are taken directly from the SQuAD dataset and one was just made up. Feel free to edit the file and add your own! + + +## To run + +* `lein install` in the root of the main project directory +* cd into this project directory and do `lein run`. This will execute the cpu version. + +`lein run :cpu` - to run with cpu +`lein run :gpu` - to run with gpu + +## Background + +To learn more about how BERT works in MXNet, please follow this [MXNet Gluon tutorial on NLP using BERT](https://medium.com/apache-mxnet/gluon-nlp-bert-6a489bdd3340). + +The model was extracted from MXNet GluonNLP with static length settings. + +[Download link for the script](https://gluon-nlp.mxnet.io/_downloads/bert.zip) + +The original description can be found in the [MXNet GluonNLP model zoo](https://gluon-nlp.mxnet.io/model_zoo/bert/index.html#bert-base-on-squad-1-1). +```bash +python static_finetune_squad.py --optimizer adam --accumulate 2 --batch_size 6 --lr 3e-5 --epochs 2 --gpu 0 --export + +``` +This script will generate `json` and `param` fles that are the standard MXNet model files. +By default, this model are using `bert_12_768_12` model with extra layers for QA jobs. + +After that, to be able to use it in Java, we need to export the dictionary from the script to parse the text +to actual indexes. Please add the following lines after [this line](https://github.com/dmlc/gluon-nlp/blob/master/scripts/bert/staticbert/static_finetune_squad.py#L262). +```python +import json +json_str = vocab.to_json() +f = open("vocab.json", "w") +f.write(json_str) +f.close() +``` +This would export the token vocabulary in json format. +Once you have these three files, you will be able to run this example without problems. + diff --git a/contrib/clojure-package/examples/bert-qa/get_bert_data.sh b/contrib/clojure-package/examples/bert-qa/get_bert_data.sh new file mode 100755 index 000000000000..603194a03c05 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/get_bert_data.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +data_path=model + +if [ ! -d "$data_path" ]; then + mkdir -p "$data_path" + curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/vocab.json -o $data_path/vocab.json + curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/static_bert_qa-0002.params -o $data_path/static_bert_qa-0002.params + curl https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/BertQA/static_bert_qa-symbol.json -o $data_path/static_bert_qa-symbol.json +fi diff --git a/contrib/clojure-package/examples/bert-qa/project.clj b/contrib/clojure-package/examples/bert-qa/project.clj new file mode 100644 index 000000000000..5bec165d75f5 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/project.clj @@ -0,0 +1,10 @@ +(defproject bert-qa "0.1.0-SNAPSHOT" + :description "BERT QA Example" + :plugins [[lein-cljfmt "0.5.7"]] + :dependencies [[org.clojure/clojure "1.9.0"] + [org.apache.mxnet.contrib.clojure/clojure-mxnet "1.5.0-SNAPSHOT"] + [cheshire "5.8.1"]] + :pedantic? :skip + :java-source-paths ["src/java"] + :main bert-qa.core + :repl-options {:init-ns bert-qa.core}) diff --git a/contrib/clojure-package/examples/bert-qa/squad-samples.edn b/contrib/clojure-package/examples/bert-qa/squad-samples.edn new file mode 100644 index 000000000000..1eb2b1321ef7 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/squad-samples.edn @@ -0,0 +1,17 @@ +[{:input-answer "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm." + :input-question "By what main attribute are computational problems classified utilizing computational complexity theory?" + :ground-truth-answers ["Computational complexity theory" + "Computational complexity theory" + "complexity theory"]} + {:input-answer "Steam engines are external combustion engines, where the working fluid is separate from the combustion products. Non-combustion heat sources such as solar power, nuclear power or geothermal energy may be used. The ideal thermodynamic cycle used to analyze this process is called the Rankine cycle. In the cycle, water is heated and transforms into steam within a boiler operating at a high pressure. When expanded through pistons or turbines, mechanical work is done. The reduced-pressure steam is then condensed and pumped back into the boiler." + :input-question "Along with geothermal and nuclear, what is a notable non-combustion heat source?" + :ground-truth-answers ["solar" + "solar power" + "solar power, nuclear power or geothermal energysolar"]} + {:input-answer "In the 1960s, a series of discoveries, the most important of which was seafloor spreading, showed that the Earth's lithosphere, which includes the crust and rigid uppermost portion of the upper mantle, is separated into a number of tectonic plates that move across the plastically deforming, solid, upper mantle, which is called the asthenosphere. There is an intimate coupling between the movement of the plates on the surface and the convection of the mantle: oceanic plate motions and mantle convection currents always move in the same direction, because the oceanic lithosphere is the rigid upper thermal boundary layer of the convecting mantle. This coupling between rigid plates moving on the surface of the Earth and the convecting mantle is called plate tectonics." + :input-question "What was the most important discovery that led to the understanding that Earth's lithosphere is separated into tectonic plates?" + :ground-truth-answers ["seafloor spreading"]} + ;;; totally made up + {:input-answer "Susan had a cat named Sammy when she lived in the green house." + :input-question "What was Susan's cat named?" + :ground-truth-answers ["Sammy" "sammy"]}] diff --git a/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj b/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj new file mode 100644 index 000000000000..a0078f563243 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj @@ -0,0 +1,144 @@ +(ns bert-qa.core + (:require [clojure.string :as string] + [clojure.reflect :as r] + [cheshire.core :as json] + [clojure.java.io :as io] + [clojure.set :as set] + [org.apache.clojure-mxnet.dtype :as dtype] + [org.apache.clojure-mxnet.context :as context] + [org.apache.clojure-mxnet.layout :as layout] + [org.apache.clojure-mxnet.ndarray :as ndarray] + [org.apache.clojure-mxnet.infer :as infer] + [clojure.pprint :as pprint])) + +(def model-path-prefix "model/static_bert_qa") +;; epoch number of the model +(def epoch 2) +;; the vocabulary used in the model +(def model-vocab "model/vocab.json") +;; the input question +;; the maximum length of the sequence +(def seq-length 384) + +;;; data helpers + +(defn break-out-punctuation [s str-match] + (->> (string/split (str s "") (re-pattern (str "\\" str-match))) + (map #(string/replace % "" str-match)))) + +(defn break-out-punctuations [s] + (if-let [target-char (first (re-seq #"[.,?!]" s))] + (break-out-punctuation s target-char) + [s])) + +(defn tokenizer [s] + (->> (string/split s #"\s+") + (mapcat break-out-punctuations) + (into []))) + +(defn pad [tokens pad-item num] + (if (>= (count tokens) num) + tokens + (into tokens (repeat (- num (count tokens)) pad-item)))) + +(defn get-vocab [] + (let [vocab (json/parse-stream (clojure.java.io/reader "model/vocab.json"))] + {:idx2token (get vocab "idx_to_token") + :token2idx (get vocab "token_to_idx")})) + +(defn tokens->idxs [token2idx tokens] + (mapv #(get token2idx % (get token2idx "[UNK]")) tokens)) + +(defn idxs->tokens [idx2token idxs] + (mapv #(get idx2token %) idxs)) + +(defn post-processing [result tokens] + (let [output1 (ndarray/slice-axis result 2 0 1) + output2 (ndarray/slice-axis result 2 1 2) + ;;; get the formatted logits result + start-logits (ndarray/reshape output1 [0 -3]) + end-logits (ndarray/reshape output2 [0 -3]) + start-prob (ndarray/softmax start-logits) + end-prob (ndarray/softmax end-logits) + start-idx (-> (ndarray/argmax start-prob 1) + (ndarray/->vec) + (first)) + end-idx (-> (ndarray/argmax end-prob 1) + (ndarray/->vec) + (first))] + (if (> end-idx start-idx) + (subvec tokens start-idx (inc end-idx)) + (subvec tokens end-idx (inc end-idx)) ) + )) + +(defn make-predictor [ctx] + (let [input-descs [{:name "data0" + :shape [1 seq-length] + :dtype dtype/FLOAT32 + :layout layout/NT} + {:name "data1" + :shape [1 seq-length] + :dtype dtype/FLOAT32 + :layout layout/NT} + {:name "data2" + :shape [1] + :dtype dtype/FLOAT32 + :layout layout/N}] + factory (infer/model-factory model-path-prefix input-descs)] + (infer/create-predictor + factory + {:contexts [ctx] + :epoch 2}))) + +(defn pre-processing [ctx idx2token token2idx qa-map] + (let [{:keys [input-question input-answer ground-truth-answers]} qa-map + ;;; pre-processing tokenize sentence + token-q (tokenizer (string/lower-case input-question)) + token-a (tokenizer (string/lower-case input-answer)) + valid-length (+ (count token-q) (count token-a)) + ;;; generate token types [0000...1111...0000] + qa-embedded (into (pad [] 0 (count token-q)) + (pad [] 1 (count token-a))) + token-types (pad qa-embedded 0 seq-length) + ;;; make BERT pre-processing standard + token-a (conj token-a "[SEP]") + token-q (into [] (concat ["[CLS]"] token-q ["[SEP]"] token-a)) + tokens (pad token-q "[PAD]" seq-length) + ;;; pre-processing - token to index translation + + indexes (tokens->idxs token2idx tokens)] + {:input-batch [(ndarray/array indexes [1 seq-length] {:context ctx}) + (ndarray/array token-types [1 seq-length] {:context ctx}) + (ndarray/array [valid-length] [1] {:context ctx})] + :tokens tokens + :qa-map qa-map})) + +(defn infer [ctx] + (let [ctx (context/default-context) + predictor (make-predictor ctx) + {:keys [idx2token token2idx]} (get-vocab) + ;;; samples taken from https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ + question-answers (clojure.edn/read-string (slurp "squad-samples.edn")) + ] + (doseq [qa-map question-answers] + (let [{:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) + result (first (infer/predict-with-ndarray predictor input-batch)) + answer (post-processing result tokens)] + (println "===============================") + (println " Question Answer Data") + (pprint/pprint qa-map) + (println) + (println " Predicted Answer: " answer) + (println "===============================") )))) + +(defn -main [& args] + (let [[dev] args] + (if (= dev ":gpu") + (infer (context/gpu)) + (infer (context/cpu))))) + +(comment + + (infer :cpu) + + ) diff --git a/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj b/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj new file mode 100644 index 000000000000..9345881801e2 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj @@ -0,0 +1,7 @@ +(ns bert-qa.core-test + (:require [clojure.test :refer :all] + [bert-qa.core :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1)))) From 8e0069ec3fbfa79f83c8c2df8904b14e3cf7014f Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 13 Apr 2019 16:06:59 -0400 Subject: [PATCH 2/6] add RAT rename core to infer add integration test --- .../examples/bert-qa/project.clj | 4 +- .../examples/bert-qa/squad-samples.edn | 18 ++++++++ .../src/bert_qa/{core.clj => infer.clj} | 32 ++++++++++---- .../bert-qa/test/bert_qa/core_test.clj | 7 ---- .../bert-qa/test/bert_qa/infer_test.clj | 42 +++++++++++++++++++ 5 files changed, 85 insertions(+), 18 deletions(-) rename contrib/clojure-package/examples/bert-qa/src/bert_qa/{core.clj => infer.clj} (84%) delete mode 100644 contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj create mode 100644 contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj diff --git a/contrib/clojure-package/examples/bert-qa/project.clj b/contrib/clojure-package/examples/bert-qa/project.clj index 5bec165d75f5..ed9f1f29f7ab 100644 --- a/contrib/clojure-package/examples/bert-qa/project.clj +++ b/contrib/clojure-package/examples/bert-qa/project.clj @@ -6,5 +6,5 @@ [cheshire "5.8.1"]] :pedantic? :skip :java-source-paths ["src/java"] - :main bert-qa.core - :repl-options {:init-ns bert-qa.core}) + :main bert-qa.infer + :repl-options {:init-ns bert-qa.infer}) diff --git a/contrib/clojure-package/examples/bert-qa/squad-samples.edn b/contrib/clojure-package/examples/bert-qa/squad-samples.edn index 1eb2b1321ef7..f974a7a3838e 100644 --- a/contrib/clojure-package/examples/bert-qa/squad-samples.edn +++ b/contrib/clojure-package/examples/bert-qa/squad-samples.edn @@ -1,3 +1,21 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + + [{:input-answer "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm." :input-question "By what main attribute are computational problems classified utilizing computational complexity theory?" :ground-truth-answers ["Computational complexity theory" diff --git a/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj b/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj similarity index 84% rename from contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj rename to contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj index a0078f563243..ee181aaad677 100644 --- a/contrib/clojure-package/examples/bert-qa/src/bert_qa/core.clj +++ b/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj @@ -1,4 +1,22 @@ -(ns bert-qa.core +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + + +(ns bert-qa.infer (:require [clojure.string :as string] [clojure.reflect :as r] [cheshire.core :as json] @@ -68,8 +86,7 @@ (first))] (if (> end-idx start-idx) (subvec tokens start-idx (inc end-idx)) - (subvec tokens end-idx (inc end-idx)) ) - )) + (subvec tokens end-idx (inc end-idx))))) (defn make-predictor [ctx] (let [input-descs [{:name "data0" @@ -118,8 +135,7 @@ predictor (make-predictor ctx) {:keys [idx2token token2idx]} (get-vocab) ;;; samples taken from https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ - question-answers (clojure.edn/read-string (slurp "squad-samples.edn")) - ] + question-answers (clojure.edn/read-string (slurp "squad-samples.edn"))] (doseq [qa-map question-answers] (let [{:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) result (first (infer/predict-with-ndarray predictor input-batch)) @@ -129,7 +145,7 @@ (pprint/pprint qa-map) (println) (println " Predicted Answer: " answer) - (println "===============================") )))) + (println "==============================="))))) (defn -main [& args] (let [[dev] args] @@ -139,6 +155,4 @@ (comment - (infer :cpu) - - ) + (infer :cpu)) diff --git a/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj b/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj deleted file mode 100644 index 9345881801e2..000000000000 --- a/contrib/clojure-package/examples/bert-qa/test/bert_qa/core_test.clj +++ /dev/null @@ -1,7 +0,0 @@ -(ns bert-qa.core-test - (:require [clojure.test :refer :all] - [bert-qa.core :refer :all])) - -(deftest a-test - (testing "FIXME, I fail." - (is (= 0 1)))) diff --git a/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj new file mode 100644 index 000000000000..4bc295f5b894 --- /dev/null +++ b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj @@ -0,0 +1,42 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + + +(ns bert-qa.infer-test + (:require [bert-qa.infer :refer :all] + [clojure.java.io :as io] + [clojure.java.shell :refer [sh]] + [clojure.test :refer :all] + [org.apache.clojure-mxnet.context :as context] + [org.apache.clojure-mxnet.infer :as infer])) + +(def model-dir "model/") + +(when-not (.exists (io/file (str model-dir "static_bert_qa-0002.params"))) + (println "Downloading bert qa data") + (sh "./get_bert_data.sh")) + +(deftest infer-test + (let [ctx (context/default-context) + predictor (make-predictor ctx) + {:keys [idx2token token2idx]} (get-vocab) + ;;; samples taken from https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ + question-answers (clojure.edn/read-string (slurp "squad-samples.edn"))] + (let [qa-map (last question-answers) + {:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) + result (first (infer/predict-with-ndarray predictor input-batch))] + (is (= ["sammy"] (post-processing result tokens)))))) From 32cb4fe8ccff5fc0a5031f03a8f0c3155620beae Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 13 Apr 2019 16:14:55 -0400 Subject: [PATCH 3/6] add rat for project.clj --- .../examples/bert-qa/project.clj | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/clojure-package/examples/bert-qa/project.clj b/contrib/clojure-package/examples/bert-qa/project.clj index ed9f1f29f7ab..d256d44d0798 100644 --- a/contrib/clojure-package/examples/bert-qa/project.clj +++ b/contrib/clojure-package/examples/bert-qa/project.clj @@ -1,3 +1,21 @@ +;; +;; Licensed to the Apache Software Foundation (ASF) under one or more +;; contributor license agreements. See the NOTICE file distributed with +;; this work for additional information regarding copyright ownership. +;; The ASF licenses this file to You under the Apache License, Version 2.0 +;; (the "License"); you may not use this file except in compliance with +;; the License. You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. +;; + + (defproject bert-qa "0.1.0-SNAPSHOT" :description "BERT QA Example" :plugins [[lein-cljfmt "0.5.7"]] From 661a2d4247539d87dacea7dfe6d56d94f58f2f96 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 13 Apr 2019 16:27:24 -0400 Subject: [PATCH 4/6] =?UTF-8?q?Couldn=E2=80=99t=20resist=20adding=20a=20qa?= =?UTF-8?q?=20about=20Clojure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- contrib/clojure-package/examples/bert-qa/squad-samples.edn | 6 +++++- .../examples/bert-qa/test/bert_qa/infer_test.clj | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/clojure-package/examples/bert-qa/squad-samples.edn b/contrib/clojure-package/examples/bert-qa/squad-samples.edn index f974a7a3838e..e99a181f7d17 100644 --- a/contrib/clojure-package/examples/bert-qa/squad-samples.edn +++ b/contrib/clojure-package/examples/bert-qa/squad-samples.edn @@ -32,4 +32,8 @@ ;;; totally made up {:input-answer "Susan had a cat named Sammy when she lived in the green house." :input-question "What was Susan's cat named?" - :ground-truth-answers ["Sammy" "sammy"]}] + :ground-truth-answers ["Sammy" "sammy"]} + ;;; more or less from wikipedia on clojure + {:input-answer "Rich Hickey is the creator of the Clojure language. Before Clojure, he developed dotLisp, a similar project based on the .NET platform, and three earlier attempts to provide interoperability between Lisp and Java: a Java foreign language interface for Common Lisp, A Foreign Object Interface for Lisp, and a Lisp-friendly interface to Java Servlets." + :input-question "Who created Clojure?" + :ground-truth-answers ["rich" "hickey"]}] diff --git a/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj index 4bc295f5b894..f78b06aeb581 100644 --- a/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj +++ b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj @@ -39,4 +39,4 @@ (let [qa-map (last question-answers) {:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) result (first (infer/predict-with-ndarray predictor input-batch))] - (is (= ["sammy"] (post-processing result tokens)))))) + (is (= ["rich" "hickey"] (post-processing result tokens)))))) From 4df5cf8d06427fc53637ecbdc526be6bb4c1dedd Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 13 Apr 2019 16:39:12 -0400 Subject: [PATCH 5/6] rat for readme --- .../clojure-package/examples/bert-qa/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/clojure-package/examples/bert-qa/README.md b/contrib/clojure-package/examples/bert-qa/README.md index a61e2709d145..ff711df103f2 100644 --- a/contrib/clojure-package/examples/bert-qa/README.md +++ b/contrib/clojure-package/examples/bert-qa/README.md @@ -1,3 +1,21 @@ + + + + + + + + + + + + + + + + + + # bert-qa **This example was based off of the Java API one. It shows how to do inference with a pre-trained BERT network that is trained on Questions and Answers using the [SQuAD Dataset](https://rajpurkar.github.io/SQuAD-explorer/)** From 8c80af5b9bde285a71742d8983262dfa01920853 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 14 Apr 2019 07:25:46 -0400 Subject: [PATCH 6/6] feedback from @kedarbellare --- .../examples/bert-qa/README.md | 2 +- .../examples/bert-qa/src/bert_qa/infer.clj | 27 ++++++++++--------- .../bert-qa/test/bert_qa/infer_test.clj | 4 +-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/contrib/clojure-package/examples/bert-qa/README.md b/contrib/clojure-package/examples/bert-qa/README.md index ff711df103f2..9a21bcdfd66b 100644 --- a/contrib/clojure-package/examples/bert-qa/README.md +++ b/contrib/clojure-package/examples/bert-qa/README.md @@ -44,7 +44,7 @@ The prediction in this case would be `solar power` For this tutorial, you can get the model and vocabulary by running following bash file. This script will use `wget` to download these artifacts from AWS S3. -From the `scala-package/examples/scripts/infer/bert/` folder run: +From the example directory: ```bash ./get_bert_data.sh diff --git a/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj b/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj index ee181aaad677..836684e04977 100644 --- a/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj +++ b/contrib/clojure-package/examples/bert-qa/src/bert_qa/infer.clj @@ -49,7 +49,7 @@ (break-out-punctuation s target-char) [s])) -(defn tokenizer [s] +(defn tokenize [s] (->> (string/split s #"\s+") (mapcat break-out-punctuations) (into []))) @@ -61,14 +61,15 @@ (defn get-vocab [] (let [vocab (json/parse-stream (clojure.java.io/reader "model/vocab.json"))] - {:idx2token (get vocab "idx_to_token") - :token2idx (get vocab "token_to_idx")})) + {:idx->token (get vocab "idx_to_token") + :token->idx (get vocab "token_to_idx")})) -(defn tokens->idxs [token2idx tokens] - (mapv #(get token2idx % (get token2idx "[UNK]")) tokens)) +(defn tokens->idxs [token->idx tokens] + (let [unk-idx (get token->idx "[UNK]")] + (mapv #(get token->idx % unk-idx) tokens))) -(defn idxs->tokens [idx2token idxs] - (mapv #(get idx2token %) idxs)) +(defn idxs->tokens [idx->token idxs] + (mapv #(get idx->token %) idxs)) (defn post-processing [result tokens] (let [output1 (ndarray/slice-axis result 2 0 1) @@ -107,11 +108,11 @@ {:contexts [ctx] :epoch 2}))) -(defn pre-processing [ctx idx2token token2idx qa-map] +(defn pre-processing [ctx idx->token token->idx qa-map] (let [{:keys [input-question input-answer ground-truth-answers]} qa-map ;;; pre-processing tokenize sentence - token-q (tokenizer (string/lower-case input-question)) - token-a (tokenizer (string/lower-case input-answer)) + token-q (tokenize (string/lower-case input-question)) + token-a (tokenize (string/lower-case input-answer)) valid-length (+ (count token-q) (count token-a)) ;;; generate token types [0000...1111...0000] qa-embedded (into (pad [] 0 (count token-q)) @@ -123,7 +124,7 @@ tokens (pad token-q "[PAD]" seq-length) ;;; pre-processing - token to index translation - indexes (tokens->idxs token2idx tokens)] + indexes (tokens->idxs token->idx tokens)] {:input-batch [(ndarray/array indexes [1 seq-length] {:context ctx}) (ndarray/array token-types [1 seq-length] {:context ctx}) (ndarray/array [valid-length] [1] {:context ctx})] @@ -133,11 +134,11 @@ (defn infer [ctx] (let [ctx (context/default-context) predictor (make-predictor ctx) - {:keys [idx2token token2idx]} (get-vocab) + {:keys [idx->token token->idx]} (get-vocab) ;;; samples taken from https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ question-answers (clojure.edn/read-string (slurp "squad-samples.edn"))] (doseq [qa-map question-answers] - (let [{:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) + (let [{:keys [input-batch tokens qa-map]} (pre-processing ctx idx->token token->idx qa-map) result (first (infer/predict-with-ndarray predictor input-batch)) answer (post-processing result tokens)] (println "===============================") diff --git a/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj index f78b06aeb581..767fb089f284 100644 --- a/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj +++ b/contrib/clojure-package/examples/bert-qa/test/bert_qa/infer_test.clj @@ -33,10 +33,10 @@ (deftest infer-test (let [ctx (context/default-context) predictor (make-predictor ctx) - {:keys [idx2token token2idx]} (get-vocab) + {:keys [idx->token token->idx]} (get-vocab) ;;; samples taken from https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/ question-answers (clojure.edn/read-string (slurp "squad-samples.edn"))] (let [qa-map (last question-answers) - {:keys [input-batch tokens qa-map]} (pre-processing ctx idx2token token2idx qa-map) + {:keys [input-batch tokens qa-map]} (pre-processing ctx idx->token token->idx qa-map) result (first (infer/predict-with-ndarray predictor input-batch))] (is (= ["rich" "hickey"] (post-processing result tokens))))))