From 047afa4d4327de28ffa0f551f2641bf22bc8cbcb Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 20:21:43 +0800 Subject: [PATCH 1/5] support pydocstyle check Signed-off-by: chensuyue --- .azure-pipelines/code-scan.yml | 16 ++++++++++-- .../docker/DockerfileCodeScan.devel | 3 ++- .../scripts/codeScan/bandit/bandit.sh | 16 +++++++----- .../scripts/codeScan/pydocstyle/pydocstyle.sh | 25 +++++++++++++++++++ .../scripts/codeScan/pylint/pylint.sh | 18 +++++++------ 5 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 .azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3c8a8c8daae..546b8f6e27d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -16,6 +16,18 @@ variables: CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" stages: + - stage: DocStyleCodeScan + displayName: DocStyle Code Scan + dependsOn: [] + jobs: + - job: DocStyle + displayName: DocStyle + steps: + - template: template/code-scan-template.yml + parameters: + codeScanFileName: "pydocstyle" + uploadPath: "pydocstyle.log" + - stage: BanditCodeScan displayName: Bandit Code Scan dependsOn: [] @@ -26,7 +38,7 @@ stages: - template: template/code-scan-template.yml parameters: codeScanFileName: "bandit" - uploadPath: "lpot-bandit.log" + uploadPath: "bandit.log" - stage: PylintCodeScan displayName: Pylint Code Scan @@ -38,7 +50,7 @@ stages: - template: template/code-scan-template.yml parameters: codeScanFileName: "pylint" - uploadPath: "lpot-pylint.json" + uploadPath: "pylint.json" - stage: CopyRight displayName: CopyRight Code Scan diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel index 93321aa0f14..8c33984f23d 100644 --- a/.azure-pipelines/docker/DockerfileCodeScan.devel +++ b/.azure-pipelines/docker/DockerfileCodeScan.devel @@ -38,6 +38,7 @@ RUN python -m pip install --no-cache-dir pylint==2.12.1\ tf_slim\ transformers\ horovod\ - flask==2.1.3 + flask==2.1.3 \ + pydocstyle WORKDIR / diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index a23f2f3000d..b8238ef5f92 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,17 +1,21 @@ #!/bin/bash source /neural-compressor/.azure-pipelines/scripts/change_color.sh -mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog -bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +RESET="echo -en \\E[0m \\n" # close color -python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log +log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +mkdir -p $log_dir + +python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $log_dir/bandit.log exit_code=$? -# code-scan close -RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo " ----------------- Current bandit cmd start --------------------------" && $RESET +echo "python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $log_dir/bandit.log" +$BOLD_YELLOW && echo " ----------------- Current bandit cmd end --------------------------" && $RESET $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" -cat $bandit_log_dir/lpot-bandit.log +cat $log_dir/bandit.log $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET diff --git a/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh b/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh new file mode 100644 index 00000000000..fc71692c37a --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +source /neural-compressor/.azure-pipelines/scripts/change_color.sh +RESET="echo -en \\E[0m \\n" # close color + +log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +mkdir -p $log_dir + +python pydocstyle --convention=google /neural-compressor/neural_compressor/experimental > $log_dir/pydocstyle.log +exit_code=$? + + +$BOLD_YELLOW && echo " ----------------- Current pydocstyle cmd start --------------------------" && $RESET +echo "python pydocstyle --convention=google /neural-compressor/neural_compressor/experimental > $log_dir/pydocstyle.log" +$BOLD_YELLOW && echo " ----------------- Current pydocstyle cmd end --------------------------" && $RESET + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat $log_dir/pydocstyle.log +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + + +if [ ${exit_code} -ne 0 ] ; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view DocStyle error details." && $RESET; exit 1 +fi +$BOLD_PURPLE && echo "Congratulations, DocStyle check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index de55395e5ce..3232a164267 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -1,24 +1,26 @@ #!/bin/bash source /neural-compressor/.azure-pipelines/scripts/change_color.sh -mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog -pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +RESET="echo -en \\E[0m \\n" # close color + +log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" +mkdir -p $log_dir pip install -r /neural-compressor/requirements.txt pip install torch==1.12.0 -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json -exit_code=$? +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \ +--ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor \ +> $log_dir/pylint.json -# code-scan close -RESET="echo -en \\E[0m \\n" +exit_code=$? $BOLD_YELLOW && echo " ----------------- Current pylint cmd start --------------------------" && $RESET -echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json" +echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $log_dir/pylint.json" $BOLD_YELLOW && echo " ----------------- Current pylint cmd end --------------------------" && $RESET $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" && $RESET -cat $pylint_log_dir/lpot-pylint.json +cat $pylint_log_dir/pylint.json $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET From b8bbfa4dc87d775b6107ca9aefd0f2c31561f702 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 20:29:02 +0800 Subject: [PATCH 2/5] issue fix Signed-off-by: chensuyue --- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 3232a164267..b15da8c91b3 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -20,7 +20,7 @@ echo "python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --ma $BOLD_YELLOW && echo " ----------------- Current pylint cmd end --------------------------" && $RESET $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" && $RESET -cat $pylint_log_dir/pylint.json +cat $log_dir/pylint.json $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET From a6360ce75fb4fb886cd3ad2204319a91d98ea4e0 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 23 Nov 2022 20:36:51 +0800 Subject: [PATCH 3/5] issue fix Signed-off-by: chensuyue --- .azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh b/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh index fc71692c37a..8b8a09939e8 100644 --- a/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh +++ b/.azure-pipelines/scripts/codeScan/pydocstyle/pydocstyle.sh @@ -6,7 +6,7 @@ RESET="echo -en \\E[0m \\n" # close color log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" mkdir -p $log_dir -python pydocstyle --convention=google /neural-compressor/neural_compressor/experimental > $log_dir/pydocstyle.log +pydocstyle --convention=google /neural-compressor/neural_compressor/experimental > $log_dir/pydocstyle.log exit_code=$? From 7f4741f30f10d36f98deb69cb5fd1979eeb889b9 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:33:44 +0800 Subject: [PATCH 4/5] add docString for public method `prepare` --- neural_compressor/experimental/pruning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neural_compressor/experimental/pruning.py b/neural_compressor/experimental/pruning.py index f005e3ee5db..7c318e38bf9 100644 --- a/neural_compressor/experimental/pruning.py +++ b/neural_compressor/experimental/pruning.py @@ -114,6 +114,7 @@ def _on_after_optimizer_step(self): pruner.on_after_optimizer_step() def prepare(self): + """Functions prepare for generate_hooks, generate_pruners.""" self.generate_hooks() self.generate_pruners() From 20c9dcf2730ee760ef6cb9a2cac80d10bc28d60b Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:36:59 +0800 Subject: [PATCH 5/5] add docString for __init__ --- neural_compressor/experimental/common/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neural_compressor/experimental/common/__init__.py b/neural_compressor/experimental/common/__init__.py index 6313abcf296..a5f07849745 100644 --- a/neural_compressor/experimental/common/__init__.py +++ b/neural_compressor/experimental/common/__init__.py @@ -1,3 +1,4 @@ +"""Intel® Neural Compressor: An open-source Python library supporting common model.""" #!/usr/bin/env python # -*- coding: utf-8 -*- #