From f23a3709fec056051cacf683407502e7a7feeab5 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 16:40:20 +0800 Subject: [PATCH 1/5] separate spell check and fix the check path Signed-off-by: chensuyue --- .azure-pipelines/code-scan.yml | 17 +++--------- .../scripts/codeScan/pyspelling/pyspelling.sh | 18 ++++++------- .../codeScan/pyspelling/pyspelling_conf.yaml | 10 ++++--- .azure-pipelines/spell-check.yml | 26 +++++++++++++++++++ 4 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 .azure-pipelines/spell-check.yml diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 2b6190b0cc2..3c8a8c8daae 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -5,7 +5,10 @@ pr: drafts: false branches: include: - - master + - master + paths: + include: + - neural_compressor pool: ICX-16C @@ -37,18 +40,6 @@ stages: codeScanFileName: "pylint" uploadPath: "lpot-pylint.json" - - stage: PyspellingCodeScan - displayName: Pyspelling Code Scan - dependsOn: [] - jobs: - - job: Pyspelling - displayName: Pyspelling - steps: - - template: template/code-scan-template.yml - parameters: - codeScanFileName: "pyspelling" - uploadPath: "lpot_pyspelling.log" - - stage: CopyRight displayName: CopyRight Code Scan dependsOn: [] diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 18f69c12d30..a5fb1cf824e 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,29 +1,27 @@ #!/bin/bash source /neural-compressor/.azure-pipelines/scripts/change_color.sh +RESET="echo -en \\E[0m \\n" # close color -mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog -pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" -pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/pyspelling" +pyspelling_log_dir="$pyspelling_dir/scanLog" +mkdir -p $pyspelling_log_dir pip install -r /neural-compressor/requirements.txt -sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${DICT_DIR}|$pyspelling_dir|g" $pyspelling_dir/pyspelling_conf.yaml +sed -i "s|\${REPO_DIR}|/neural-compressor|g" $pyspelling_dir/pyspelling_conf.yaml -pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +pyspelling -c $pyspelling_dir/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log exit_code=$? -# code-scan close -RESET="echo -en \\E[0m \\n" - $BOLD_YELLOW && echo "------------------- Current log file output start --------------------------" && $RESET cat $pyspelling_log_dir/lpot_pyspelling.log $BOLD_YELLOW && echo "------------------- Current log file output end ----------------------------" && $RESET if [ ${exit_code} -ne 0 ] ; then - $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pyspelling error details." && $RESET; exit 1 + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view pyspelling error details." && $RESET; exit 1 fi $BOLD_PURPLE && echo "Congratulations, Pyspelling check passed!" && $LIGHT_PURPLE && echo "You can click on the artifact button to see the log details." && $RESET; exit 0 diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml index c407f846ea2..20d30e6975e 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml @@ -4,8 +4,10 @@ matrix: d: en_US.ISO8859-15 dictionary: wordlists: - - ${VAL_REPO}/pyspelling/lpot_dict.txt - output: ${VAL_REPO}/pyspelling/lpot_dict.dic + - ${DICT_DIR}/lpot_dict.txt + output: ${DICT_DIR}/lpot_dict.dic sources: - - ${LPOT_REPO}/neural_coder/**/*.md - - ${LPOT_REPO}/neural_coder/*.md \ No newline at end of file + - ${REPO_DIR}/docs/* + - ${REPO_DIR}/*.md + - ${REPO_DIR}/examples/**/*.md|!${REPO_DIR}/examples/pytorch/**/huggingface_models/**/*.md + - ${REPO_DIR}/neural_compressor/**/*.md \ No newline at end of file diff --git a/.azure-pipelines/spell-check.yml b/.azure-pipelines/spell-check.yml new file mode 100644 index 00000000000..daf1006f5ee --- /dev/null +++ b/.azure-pipelines/spell-check.yml @@ -0,0 +1,26 @@ +trigger: none + +pr: + autoCancel: true + drafts: false + branches: + include: + - master + +pool: ICX-16C + +variables: + CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" + +stages: + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - template: template/code-scan-template.yml + parameters: + codeScanFileName: "pyspelling" + uploadPath: "lpot_pyspelling.log" \ No newline at end of file From 1864c09ef165e4dbb903b1c6c366e9e8eac0ab5e Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 17:01:12 +0800 Subject: [PATCH 2/5] support neural coder spell scan as well Signed-off-by: chensuyue --- .../scripts/codeScan/pyspelling/pyspelling_conf.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml index 20d30e6975e..07fe82c07f3 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml @@ -10,4 +10,6 @@ matrix: - ${REPO_DIR}/docs/* - ${REPO_DIR}/*.md - ${REPO_DIR}/examples/**/*.md|!${REPO_DIR}/examples/pytorch/**/huggingface_models/**/*.md - - ${REPO_DIR}/neural_compressor/**/*.md \ No newline at end of file + - ${REPO_DIR}/neural_compressor/**/*.md + - ${REPO_DIR}/neural_coder/**/*.md + - ${REPO_DIR}/neural_coder/*.md \ No newline at end of file From d4d1594ba066b05ea865469ba37339357fbf71ed Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 17:03:33 +0800 Subject: [PATCH 3/5] update dict Signed-off-by: chensuyue --- .azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt index 4bb3fc06e39..31999848eee 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -2366,3 +2366,4 @@ christmas dicoo snowly syncedreview +ww From cbef7ed12c67db62cb6f82f10db07e5f38b8e13d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 17:09:15 +0800 Subject: [PATCH 4/5] fix log path Signed-off-by: chensuyue --- .../scripts/codeScan/pyspelling/pyspelling.sh | 14 +++++++------- .azure-pipelines/spell-check.yml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index a5fb1cf824e..7ae720edf0b 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -3,20 +3,20 @@ source /neural-compressor/.azure-pipelines/scripts/change_color.sh RESET="echo -en \\E[0m \\n" # close color -pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/pyspelling" -pyspelling_log_dir="$pyspelling_dir/scanLog" -mkdir -p $pyspelling_log_dir +work_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/pyspelling" +log_dir="$work_dir/../scanLog" +mkdir -p $log_dir pip install -r /neural-compressor/requirements.txt -sed -i "s|\${DICT_DIR}|$pyspelling_dir|g" $pyspelling_dir/pyspelling_conf.yaml -sed -i "s|\${REPO_DIR}|/neural-compressor|g" $pyspelling_dir/pyspelling_conf.yaml +sed -i "s|\${DICT_DIR}|$work_dir|g" $work_dir/pyspelling_conf.yaml +sed -i "s|\${REPO_DIR}|/neural-compressor|g" $work_dir/pyspelling_conf.yaml -pyspelling -c $pyspelling_dir/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +pyspelling -c $work_dir/pyspelling_conf.yaml > $log_dir/pyspelling.log exit_code=$? $BOLD_YELLOW && echo "------------------- Current log file output start --------------------------" && $RESET -cat $pyspelling_log_dir/lpot_pyspelling.log +cat $log_dir/pyspelling.log $BOLD_YELLOW && echo "------------------- Current log file output end ----------------------------" && $RESET diff --git a/.azure-pipelines/spell-check.yml b/.azure-pipelines/spell-check.yml index daf1006f5ee..011dfa8bbe1 100644 --- a/.azure-pipelines/spell-check.yml +++ b/.azure-pipelines/spell-check.yml @@ -23,4 +23,4 @@ stages: - template: template/code-scan-template.yml parameters: codeScanFileName: "pyspelling" - uploadPath: "lpot_pyspelling.log" \ No newline at end of file + uploadPath: "pyspelling.log" \ No newline at end of file From 60aeb537bd701c173580c2d251fdc0c0002a5b63 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 17:30:46 +0800 Subject: [PATCH 5/5] update dict Signed-off-by: chensuyue --- .azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt | 7 +++++++ docs/adaptor.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt index 31999848eee..3eea4060374 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -2367,3 +2367,10 @@ dicoo snowly syncedreview ww +EmbedLayerNormalization +FusedConv +GlobalAveragePool +LeakyRelu +ONNXRTAdaptor +QLinearOpsAdaptor +onnxrtadaptor diff --git a/docs/adaptor.md b/docs/adaptor.md index 73a2523ace6..5a1a31f0c7f 100644 --- a/docs/adaptor.md +++ b/docs/adaptor.md @@ -57,7 +57,7 @@ class AbcAdaptor(Adaptor): ``` * `quantize` function is used to perform quantization for post-training quantization and quantization-aware training. Quantization processing includes calibration and conversion processing for post-training quantization, while for quantization-aware training, it includes training and conversion processing. -* `evaluate` function is used to run an evaluation on a validation dataset. It is a built-in function, if user wants to use specifical evaluation function, he can pass the evaluation function to quantizer. +* `evaluate` function is used to run an evaluation on a validation dataset. It is a built-in function, if user wants to use specific evaluation function, he can pass the evaluation function to quantizer. * `query_fw_capability` function is used to run a query framework quantization capability and intersects with the user yaml configuration. * `query_fused_patterns` function is used to run a query framework graph fusion capability and decide the fusion tuning space.