Skip to content

Enable mypy on Python #1041

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 4 commits into from
May 21, 2020
Merged

Enable mypy on Python #1041

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f466aaa
Enable mypy on the repository
Fokko May 13, 2020
4356d1a
Add type annotation on field method
Fokko May 13, 2020
6b8a1f1
Add --ignore-missing-imports
Fokko May 21, 2020
9f11cd4
Merge branch 'master' of https://github.com/apache/incubator-iceberg …
Fokko May 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,6 @@ bin/

# Hive/metastore files
metastore_db/

# Python stuff
python/.mypy_cache/
10 changes: 5 additions & 5 deletions python/iceberg/api/types/type_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.

import math
from typing import List

from .type import (Type,
TypeID)
Expand Down Expand Up @@ -238,7 +239,6 @@ def get(self):
return self.visitor.field(self.field, VisitFuture(self.field.type, self.visitor).get)


@staticmethod
def decimal_required_bytes(precision):
if precision < 0 or precision > 40:
raise RuntimeError("Unsupported decimal precision: %s" % precision)
Expand Down Expand Up @@ -451,11 +451,11 @@ def write_compatibility_errors(read_schema, write_schema):
def read_compatibility_errors(read_schema, write_schema):
visit(write_schema, CheckCompatibility(read_schema, False))

NO_ERRORS = []
NO_ERRORS: List[str] = []
Copy link
Contributor

@rdblue rdblue May 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this also be a tuple?

Copy link
Contributor Author

@Fokko Fokko May 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it will actually return a list:
https://github.com/apache/incubator-iceberg/pull/1041/files/f466aaadac944166c24b2326575dbad47f242fc9#diff-36b662d03c587f1eb18d820c6b5f6171R504

Copy link
Contributor

@rdblue rdblue May 13, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I meant was: should this be tuple() because it is a constant and should not be mutable?


def __init__(self, schema, check_ordering):
self.schema = schema
self.check_ordering
self.check_ordering = check_ordering
self.current_type = None

def schema(self, schema, struct_result):
Expand Down Expand Up @@ -498,10 +498,10 @@ def struct(self, struct, field_results):

return errors

def field(self, field, field_result):
def field(self, field, field_result) -> List[str]:
struct = self.current_type.as_struct_type()
curr_field = struct.field(field.field_id)
errors = list()
errors = []

if curr_field is None:
if not field.is_optional:
Expand Down
4 changes: 2 additions & 2 deletions python/iceberg/core/base_table_scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@

class BaseTableScan(CloseableGroup, TableScan):
DATE_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
SNAPSHOT_COLUMNS = ["snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
SNAPSHOT_COLUMNS = ("snapshot_id", "file_path", "file_ordinal", "file_format", "block_size_in_bytes",
"file_size_in_bytes", "record_count", "partition", "value_counts", "null_value_counts",
"lower_bounds", "upper_bounds"]
"lower_bounds", "upper_bounds")

def new_refined_scan(self, ops, table, schema, snapshot_id, row_filter,
case_sensitive, selected_columns, options, minused_cols):
Expand Down
4 changes: 2 additions & 2 deletions python/iceberg/core/manifest_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@


class ManifestReader(CloseableGroup, Filterable):
ALL_COLUMNS = ["*"]
CHANGE_COLUMNS = ["file_path", "file_format", "partition", "record_count", "file_size_in_bytes"]
ALL_COLUMNS = ("*",)
CHANGE_COLUMNS = ("file_path", "file_format", "partition", "record_count", "file_size_in_bytes")

@staticmethod
def read(file, spec_lookup=None):
Expand Down
13 changes: 10 additions & 3 deletions python/tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,11 @@ deps =
.
{[testenv:flake8]deps}
{[testenv:bandit]deps}
{[testenv:mypy]deps}
commands =
{[testenv:flake8]commands}
{[testenv:bandit]commands}
{[testenv:mypy]commands}

[testenv:flake8]
basepython = python3
Expand All @@ -53,6 +55,14 @@ deps =
commands =
flake8 iceberg setup.py tests

[testenv:mypy]
basepython = python3
skip_install = true
deps =
mypy
commands =
mypy --ignore-missing-imports iceberg/

[testenv:bandit]
basepython = python3
skip_install = true
Expand All @@ -77,9 +87,6 @@ commands =
# commands =
# python -m http.server {posargs}

[bandit]
skips = B104

[flake8]
ignore = E501,W503
exclude =
Expand Down