diff --git a/python/.pre-commit-config.yaml b/python/.pre-commit-config.yaml index e2e25c84bfff..8160c428e1ff 100644 --- a/python/.pre-commit-config.yaml +++ b/python/.pre-commit-config.yaml @@ -50,6 +50,11 @@ repos: hooks: - id: pyupgrade args: [--py38-plus] + - repo: https://github.com/pycqa/pylint + rev: v2.14.2 + hooks: + - id: pylint + args: [ --rcfile=python/pylintrc ] - repo: https://github.com/pycqa/flake8 rev: '4.0.1' hooks: diff --git a/python/poetry.lock b/python/poetry.lock index 91dfbc691400..bc47c6e866b9 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -41,7 +41,7 @@ python-versions = ">=3.6.1" [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "dev" optional = false @@ -460,8 +460,8 @@ cfgv = [ {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] coverage = [ {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, diff --git a/python/pylintrc b/python/pylintrc index b9aa99a6ff43..983553520915 100644 --- a/python/pylintrc +++ b/python/pylintrc @@ -101,7 +101,7 @@ disable=all # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. -enable=spelling,W +enable=W [REPORTS] @@ -151,50 +151,6 @@ logging-format-style=old # function parameter format. logging-modules=logging - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: af (aspell), am (aspell), -# ar (aspell), ast (aspell), az (aspell), be (aspell), be_BY (aspell), be_SU -# (aspell), bg (aspell), bn (aspell), br (aspell), ca (aspell), cs (aspell), -# csb (aspell), cy (aspell), da (aspell), de (aspell), de_AT (aspell), de_CH -# (aspell), de_DE (aspell), el (aspell), en (aspell), en_AU (aspell), en_CA -# (aspell), en_GB (aspell), en_US (aspell), eo (aspell), es (aspell), es_ES -# (AppleSpell), et (aspell), fa (aspell), fi (aspell), fo (aspell), fr -# (aspell), fr_CH (aspell), fr_FR (aspell), fy (aspell), ga (aspell), gd -# (aspell), gl (aspell), gr (aspell), grc (aspell), gu (aspell), gv (aspell), -# he (aspell), hi (aspell), hil (aspell), hr (aspell), hsb (aspell), hu -# (aspell), hu_HU (AppleSpell), hus (aspell), hy (aspell), ia (aspell), id -# (aspell), it (aspell), it_IT (AppleSpell), kn (aspell), ku (aspell), ky -# (aspell), la (aspell), lt (aspell), lv (aspell), mg (aspell), mi (aspell), mk -# (aspell), ml (aspell), mn (aspell), mr (aspell), ms (aspell), mt (aspell), -# nds (aspell), nl (aspell), nl_NL (AppleSpell), nn (aspell), ny (aspell), or -# (aspell), pa (aspell), pl (aspell), pt_BR (aspell), pt_PT (aspell), qu -# (aspell), ro (aspell), ru (aspell), rw (aspell), sc (aspell), sk (aspell), -# sk_SK (aspell), sl (aspell), sr (aspell), srd (aspell), sv (aspell), sv_SE -# (AppleSpell), sw (aspell), ta (aspell), te (aspell), tet (aspell), tk -# (aspell), tl (aspell), tn (aspell), tr (aspell), uk (aspell), uz (aspell), vi -# (aspell), wa (aspell), yi (aspell), zu (aspell). -spelling-dict=en_US - -# List of comma separated words that should be considered directives if they -# appear and the beginning of a comment and should not be checked. -spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file=spellcheck-dictionary.txt - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. diff --git a/python/pyproject.toml b/python/pyproject.toml index afeee351ce52..5d7b75f868a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -40,7 +40,6 @@ packages = [ [tool.poetry.dependencies] python = "^3.8" mmh3 = "^3.0.0" - pyarrow = { version = "^8.0.0", optional = true } zstandard = { version = "^0.17.0", optional = true } diff --git a/python/spellcheck-dictionary.txt b/python/spellcheck-dictionary.txt deleted file mode 100644 index 8bb0b868c388..000000000000 --- a/python/spellcheck-dictionary.txt +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -accessor -accessors -Args -Avro -Nestedfield -ASF -BD -bool -boolean -datetime -disjunction -dispatchable -endian -enum -FileInfo -filesystem -fs -func -IcebergType -io -namespace -namespaces -NativeFile -NaN -nan -NestedField -nullability -PartitionField -pragma -PrimitiveType -pyarrow -repr -schemas -seekable -singledispatch -str -struct -StructProtocol -StructType -Timestamptz -Timestamptzs -unscaled -URI -UnboundPredicate -BoundPredicate -BooleanExpression -BooleanExpressionVisitor -zigzag -unix -zlib -Codecs -codecs -uri diff --git a/python/src/iceberg/avro/decoder.py b/python/src/iceberg/avro/decoder.py index 24312cdd250f..586aabf97814 100644 --- a/python/src/iceberg/avro/decoder.py +++ b/python/src/iceberg/avro/decoder.py @@ -103,7 +103,7 @@ def read_decimal_from_bytes(self, precision: int, scale: int) -> decimal.Decimal size = self.read_long() return self.read_decimal_from_fixed(precision, scale, size) - def read_decimal_from_fixed(self, precision: int, scale: int, size: int) -> decimal.Decimal: + def read_decimal_from_fixed(self, _: int, scale: int, size: int) -> decimal.Decimal: """ Decimal is encoded as fixed. Fixed instances are encoded using the number of bytes declared in the schema. diff --git a/python/src/iceberg/avro/reader.py b/python/src/iceberg/avro/reader.py index bd8f1b4e8087..012d611a1a51 100644 --- a/python/src/iceberg/avro/reader.py +++ b/python/src/iceberg/avro/reader.py @@ -26,7 +26,8 @@ from __future__ import annotations from abc import abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass +from dataclasses import field as dataclassfield from datetime import date, datetime, time from decimal import Decimal from functools import singledispatch @@ -61,7 +62,7 @@ @dataclass(frozen=True) class AvroStruct(StructProtocol): - _data: list[Any | StructProtocol] = field() + _data: list[Any | StructProtocol] = dataclassfield() def set(self, pos: int, value: Any) -> None: self._data[pos] = value @@ -138,7 +139,7 @@ def read(self, decoder: BinaryDecoder) -> UUID: @dataclass(frozen=True) class FixedReader(Reader): - length: int = field() + length: int = dataclassfield() def read(self, decoder: BinaryDecoder) -> bytes: return decoder.read(self.length) @@ -151,8 +152,8 @@ def read(self, decoder: BinaryDecoder) -> bytes: @dataclass(frozen=True) class DecimalReader(Reader): - precision: int = field() - scale: int = field() + precision: int = dataclassfield() + scale: int = dataclassfield() def read(self, decoder: BinaryDecoder) -> Decimal: return decoder.read_decimal_from_bytes(self.precision, self.scale) @@ -160,7 +161,7 @@ def read(self, decoder: BinaryDecoder) -> Decimal: @dataclass(frozen=True) class OptionReader(Reader): - option: Reader = field() + option: Reader = dataclassfield() def read(self, decoder: BinaryDecoder) -> Any | None: # For the Iceberg spec it is required to set the default value to null @@ -179,7 +180,7 @@ def read(self, decoder: BinaryDecoder) -> Any | None: @dataclass(frozen=True) class StructReader(Reader): - fields: tuple[Reader, ...] = field() + fields: tuple[Reader, ...] = dataclassfield() def read(self, decoder: BinaryDecoder) -> AvroStruct: return AvroStruct([field.read(decoder) for field in self.fields]) diff --git a/python/src/iceberg/transforms.py b/python/src/iceberg/transforms.py index fbbdd917aa02..0947cf56b590 100644 --- a/python/src/iceberg/transforms.py +++ b/python/src/iceberg/transforms.py @@ -281,23 +281,23 @@ def _(self, value: int) -> str: return self._int_to_human_string(self._type, value) @singledispatchmethod - def _int_to_human_string(self, value_type: IcebergType, value: int) -> str: + def _int_to_human_string(self, _: IcebergType, value: int) -> str: return str(value) @_int_to_human_string.register(DateType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_day(value) @_int_to_human_string.register(TimeType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_time(value) @_int_to_human_string.register(TimestampType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_timestamp(value) @_int_to_human_string.register(TimestamptzType) - def _(self, value_type: IcebergType, value: int) -> str: + def _(self, _: IcebergType, value: int) -> str: return datetime.to_human_timestamptz(value) diff --git a/python/tests/avro/test_reader.py b/python/tests/avro/test_reader.py index c310e69204cc..e8c56f122e84 100644 --- a/python/tests/avro/test_reader.py +++ b/python/tests/avro/test_reader.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=protected-access import json import pytest diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 48f3bf02390b..782316ec9192 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint:disable=redefined-outer-name from tempfile import TemporaryDirectory from typing import Any, Dict diff --git a/python/tests/io/test_io_base.py b/python/tests/io/test_io_base.py index 9008c453a2ab..a4ffe7dff96a 100644 --- a/python/tests/io/test_io_base.py +++ b/python/tests/io/test_io_base.py @@ -124,7 +124,7 @@ def delete(self, location: Union[str, InputFile, OutputFile]) -> None: try: os.remove(parsed_location.path) except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path} - Caused by: {e}") from e + raise FileNotFoundError(f"Cannot delete file, does not exist: {parsed_location.path}") from e @pytest.mark.parametrize("CustomInputFile", [LocalInputFile, PyArrowFile]) diff --git a/python/tests/io/test_pyarrow.py b/python/tests/io/test_pyarrow.py index 79e88f3e24ff..f3a41bb54d40 100644 --- a/python/tests/io/test_pyarrow.py +++ b/python/tests/io/test_pyarrow.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=protected-access,unused-argument import os import tempfile diff --git a/python/tests/utils/test_schema_conversion.py b/python/tests/utils/test_schema_conversion.py index 234ec1573f46..e2eada5c40b9 100644 --- a/python/tests/utils/test_schema_conversion.py +++ b/python/tests/utils/test_schema_conversion.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=W0212 from typing import Any, Dict import pytest