Refine TF NLP models with INC User NewAPI

.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt

@@ -2379,3 +2379,7 @@ grappler
 amsgrad
 qoperator
 apis
+AccuracyCriterion
+AccuracyLoss
+PostTrainingQuantConfig
+TuningCriterion

examples/tensorflow/nlp/bert_base_mrpc/quantization/ptq/run_classifier.py

@@ -1071,6 +1071,7 @@ def result(self):
 
       eval_examples = processor.get_dev_examples(FLAGS.data_dir)
       eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
+      dataset = Dataset(eval_file, FLAGS.eval_batch_size)
 
       convert_examples_to_features(
           examples=eval_examples,
@@ -1085,19 +1086,54 @@ def result(self):
           is_training=False,
           drop_remainder=False)
 
-      from neural_compressor.experimental import Quantization, common
-      quantizer = Quantization(FLAGS.config)
-      dataset = Dataset(eval_file, FLAGS.eval_batch_size)
-      quantizer.model = common.Model(estimator, input_fn=estimator_input_fn)
-      quantizer.calib_dataloader = common.DataLoader(dataset, collate_fn=collate_fn)
-      quantizer.eval_dataloader = common.DataLoader(dataset, collate_fn=collate_fn)
-      quantizer.metric = Accuracy()
-      q_model = quantizer.fit()
+      from neural_compressor.experimental import common
+      from neural_compressor.quantization import fit
+      from neural_compressor.config import PostTrainingQuantConfig, \
+          TuningCriterion, AccuracyCriterion, AccuracyLoss, set_random_seed
+
+      set_random_seed(9527)
+
+      tuning_criterion = TuningCriterion(
+          strategy="basic",
+          timeout=0,
+          max_trials=100,
+          objective="performance")
+
+      tolerable_loss = AccuracyLoss(loss=0.01)
+
+      accuracy_criterion = AccuracyCriterion(
+          higher_is_better=True,
+          criterion='relative',
+          tolerable_loss=tolerable_loss)
+
+      config = PostTrainingQuantConfig(
+          device="cpu",
+          backend="tensorflow",
+          inputs=["input_file", "batch_size"],
+          outputs=["loss/Softmax:0", "IteratorGetNext:3"],
+          approach="static",
+          calibration_sampling_size=[500],
+          op_type_list=None,
+          op_name_list=None,
+          reduce_range=None,
+          extra_precisions=[],
+          tuning_criterion=tuning_criterion,
+          accuracy_criterion=accuracy_criterion)
+
+      q_model = fit(
+          model=common.Model(estimator, input_fn=estimator_input_fn),
+          conf=config,
+          calib_dataloader=common.DataLoader(dataset, collate_fn=collate_fn),
+          calib_func=None,
+          eval_dataloader=common.DataLoader(dataset, collate_fn=collate_fn),
+          eval_func=None,
+          eval_metric=Accuracy())
+
       if FLAGS.strip_iterator:
           q_model.graph_def = strip_iterator(q_model.graph_def)
       q_model.save(FLAGS.output_model)
 
-  if FLAGS.benchmark:
+  if FLAGS.benchmark and FLAGS.mode=="accuracy":
       eval_examples = processor.get_dev_examples(FLAGS.data_dir)
       eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
 
@@ -1120,8 +1156,82 @@ def result(self):
               is_training=False,
               drop_remainder=False)
           evaluator.model = common.Model(estimator, input_fn=estimator_input_fn)
-      evaluator(FLAGS.mode)
+      evaluator("accuracy")
+
+      """ Refactor code of benchmark in accuracy mode with INC User NewAPI.
+      from neural_compressor.experimental import common
+      from neural_compressor.benchmark import fit
+      from neural_compressor.config import BenchmarkConfig
+      from neural_compressor.model.model import get_model_type
+
+      model_type = get_model_type(FLAGS.input_model)
+      if model_type == 'frozen_pb':
+          model = FLAGS.input_model
+      else:
+          estimator_input_fn = input_fn_builder(
+              input_file=eval_file,
+              seq_length=FLAGS.max_seq_length,
+              is_training=False,
+              drop_remainder=False)
+          model = common.Model(estimator, input_fn=estimator_input_fn)
+
+      dataset = Dataset(eval_file, FLAGS.eval_batch_size)
+      b_dataloader = common.DataLoader(
+          dataset,
+          batch_size=FLAGS.eval_batch_size,
+          collate_fn=collate_fn)
+
+      config = BenchmarkConfig(
+          warmup=5,
+          iteration=20,
+          cores_per_instance=28,
+          num_of_instance=1,
+          inter_num_of_threads=None,
+          intra_num_of_threads=None)
+
+      fit(model=model, config=config, b_dataloader=b_dataloader, b_func=b_func)
+      """
+      # Cannot pass 'metric' and 'mode' in fit.
+      # TODO: implement a b_func.
+
+  elif FLAGS.benchmark and FLAGS.mode=="performance":
+      from neural_compressor.experimental import common
+      from neural_compressor.benchmark import fit
+      from neural_compressor.config import BenchmarkConfig
+      from neural_compressor.model.model import get_model_type
+
+      eval_examples = processor.get_dev_examples(FLAGS.data_dir)
+      eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record")
 
+      model_type = get_model_type(FLAGS.input_model)
+      if model_type == 'frozen_pb':
+          model = FLAGS.input_model
+      else:
+          estimator_input_fn = input_fn_builder(
+              input_file=eval_file,
+              seq_length=FLAGS.max_seq_length,
+              is_training=False,
+              drop_remainder=False)
+          model = common.Model(estimator, input_fn=estimator_input_fn)
+
+      dataset = Dataset(eval_file, FLAGS.eval_batch_size)
+      b_dataloader = common.DataLoader(
+          dataset,
+          batch_size=FLAGS.eval_batch_size,
+          collate_fn=collate_fn)
+
+      config = BenchmarkConfig(
+          inputs=["input_file", "batch_size"],
+          outputs=["loss/Softmax:0" ,"IteratorGetNext:3"],
+          warmup=5,
+          iteration=20,
+          cores_per_instance=28,
+          num_of_instance=1,
+          inter_num_of_threads=None,
+          intra_num_of_threads=None)
+
+      fit(model=model, config=config, b_dataloader=b_dataloader, b_func=None)
+
 
 if __name__ == "__main__":
   flags.mark_flag_as_required("data_dir")

examples/tensorflow/nlp/distilbert_base/quantization/ptq/README.md

@@ -55,7 +55,6 @@ bash run_tuning.sh \
     --input_model=$INPUT_MODEL \
     --dataset_location=$DATASET_DIR \
     --output_model=$OUTPUT_MODEL \
-    --config=$CONFIG_FILE \
     --batch_size=$BATCH_SIZE \
     --max_seq_length=$MAX_SEQ \
     --warmup_steps=$WARMUPS \
@@ -94,7 +93,6 @@ Where (Default values are shown in the square brackets):
    * $INPUT_MODEL ["./distilbert_base_fp32.pb"]-- The path to input FP32 frozen model .pb file to load
    * $DATASET_DIR ["./sst2_validation_dataset"]-- The path to input dataset directory
    * $OUTPUT_MODEL ["./output_distilbert_base_int8.pb"]-- The user-specified export path to the output INT8 quantized model
-   * $CONFIG_FILE ["./distilbert_base.yaml"]-- The path to quantization configuration .yaml file to load for tuning
    * $BATCH_SIZE [128]-- The batch size for model inference
    * $MAX_SEQ [128]-- The maximum total sequence length after tokenization
    * $ITERS [872]-- The number of iterations to run in benchmark mode, maximum value is 872
@@ -108,7 +106,7 @@ Details of enabling Intel® Neural Compressor on DistilBERT base for TensorFlow
 
 This is a tutorial of how to enable DistilBERT base model with Intel® Neural Compressor.
 ## User Code Analysis
-1. User specifies fp32 *model*, calibration dataloader *q_dataloader*, evaluation dataloader *eval_dataloader* and metric in tuning.metric field of model-specific yaml config file.
+1. User specifies fp32 *model*, calibration dataloader *q_dataloader*, evaluation dataloader *eval_dataloader* and metric in tuning.metric field of model-specific config.
 
 2. User specifies fp32 *model*, calibration dataloader *q_dataloader* and a custom *eval_func* which encapsulates the evaluation dataloader and metric by itself.
 
@@ -138,45 +136,50 @@ class Dataloader(object):
             yield feed_dict, labels
 ```
 
-### Write Yaml Config File
-In examples directory, there is a distilbert_base.yaml for tuning the model on Intel CPUs. The 'framework' in the yaml is set to 'tensorflow'. If running this example on Intel GPUs, the 'framework' should be set to 'tensorflow_itex' and the device in yaml file should be set to 'gpu'. The distilbert_base_itex.yaml is prepared for the GPU case. We could remove most of items and only keep mandatory item for tuning. We also implement a calibration dataloader and have evaluation field for creation of evaluation function at internal neural_compressor.
-
-```yaml
-model:
-  name: distilbert_base
-  framework: tensorflow
-
-device: cpu                # optional. default value is cpu, other value is gpu.
-
-quantization:
-  calibration:
-    sampling_size: 500
-  model_wise:
-    weight:
-      granularity: per_channel
-
-tuning:
-  accuracy_criterion:
-    relative: 0.02
-  exit_policy:
-    timeout: 0
-    max_trials: 100
-    performance_only: False
-  random_seed: 9527
-```
-
-In this case we calibrate and quantize the model, and use our user-defined calibration dataloader.
-
 ### Code Update
 After prepare step is done, we add the code for quantization tuning to generate quantized model.
 
 ```python
-from neural_compressor.experimental import Quantization, common
-quantizer = Quantization(ARGS.config)
-quantizer.calib_dataloader = self.dataloader
-quantizer.model = common.Model(graph)
-quantizer.eval_func = self.eval_func 
-q_model = quantizer.fit()
+from neural_compressor.quantization import fit
+from neural_compressor.config import PostTrainingQuantConfig, \
+    TuningCriterion, AccuracyCriterion, AccuracyLoss, set_random_seed
+
+set_random_seed(9527)
+tuning_criterion = TuningCriterion(
+    strategy="basic",
+    timeout=0,
+    max_trials=100,
+    objective="performance")
+
+tolerable_loss = AccuracyLoss(loss=0.02)
+
+accuracy_criterion = AccuracyCriterion(
+    higher_is_better=True,
+    criterion='relative',
+    tolerable_loss=tolerable_loss)
+
+config = PostTrainingQuantConfig(
+    device="cpu",
+    backend="tensorflow",
+    inputs=[],
+    outputs=[],
+    approach="static",
+    calibration_sampling_size=[500],
+    op_type_list=None,
+    op_name_list=None,
+    reduce_range=None,
+    extra_precisions=[],
+    tuning_criterion=tuning_criterion,
+    accuracy_criterion=accuracy_criterion)
+
+q_model = fit(
+    model=graph,
+    conf=config,
+    calib_dataloader=self.dataloader,
+    calib_func=None,
+    eval_dataloader=None,
+    eval_func=self.eval_func,
+    eval_metric=None)
 ```
 
 The Intel® Neural Compressor quantizer.fit() function will return a best quantized model under time constraint.

examples/tensorflow/nlp/distilbert_base/quantization/ptq/run_inference.py

@@ -1,3 +1,4 @@
+
 #
 # -*- coding: utf-8 -*-
 #
@@ -25,7 +26,9 @@
 from transformers import AutoTokenizer
 from datasets import load_from_disk
 from tensorflow.core.protobuf import saved_model_pb2
-from neural_compressor.experimental import Quantization, common
+from neural_compressor.quantization import fit
+from neural_compressor.config import PostTrainingQuantConfig, \
+    TuningCriterion, AccuracyCriterion, AccuracyLoss, set_random_seed
 from neural_compressor.utils.utility import dump_elapsed_time
 from neural_compressor.utils import logger
 
@@ -259,11 +262,43 @@ def eval_func(self, graph):
     def run(self):
         graph = self.load_graph()
         if ARGS.mode == "tune":
-            quantizer = Quantization(ARGS.config)
-            quantizer.calib_dataloader = self.dataloader
-            quantizer.model = common.Model(graph)
-            quantizer.eval_func = self.eval_func 
-            q_model = quantizer.fit()
+            set_random_seed(9527)
+            tuning_criterion = TuningCriterion(
+                strategy="basic",
+                timeout=0,
+                max_trials=100,
+                objective="performance")
+
+            tolerable_loss = AccuracyLoss(loss=0.02)
+
+            accuracy_criterion = AccuracyCriterion(
+                higher_is_better=True,
+                criterion='relative',
+                tolerable_loss=tolerable_loss)
+
+            config = PostTrainingQuantConfig(
+                device="cpu",
+                backend="tensorflow",
+                inputs=[],
+                outputs=[],
+                approach="static",
+                calibration_sampling_size=[500],
+                op_type_list=None,
+                op_name_list=None,
+                reduce_range=None,
+                extra_precisions=[],
+                tuning_criterion=tuning_criterion,
+                accuracy_criterion=accuracy_criterion)
+
+            q_model = fit(
+                model=graph,
+                conf=config,
+                calib_dataloader=self.dataloader,
+                calib_func=None,
+                eval_dataloader=None,
+                eval_func=self.eval_func,
+                eval_metric=None)
+
             try:
                 q_model.save(ARGS.output_graph)
             except Exception as e:

neural_compressor/adaptor/tensorflow.py

@@ -247,6 +247,7 @@ def evaluate(self, model, dataloader, postprocess=None,
         import tensorflow as tf
         from .tf_utils.util import iterator_sess_run
         outputs = model.output_tensor_names
+        iteration=-1 if iteration is None else iteration
 
         if getattr(dataloader, 'distributed', False):
             import horovod.tensorflow as hvd