-
-
Notifications
You must be signed in to change notification settings - Fork 38.2k
Make typing checks more strict #14429
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
41ef7e7
0d472d7
0c3a591
14afc72
2ffe76b
821fcce
d55cec7
d80f613
35b1c3f
55a69fa
61c5886
0bd1259
193d12f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,9 +6,11 @@ | |
| from typing import Any, Dict, Union, Optional, Tuple # NOQA | ||
|
|
||
| import pytz | ||
| import pytz.exceptions as pytzexceptions | ||
|
|
||
| DATE_STR_FORMAT = "%Y-%m-%d" | ||
| UTC = DEFAULT_TIME_ZONE = pytz.utc # type: dt.tzinfo | ||
| UTC = pytz.utc | ||
| DEFAULT_TIME_ZONE = pytz.utc # type: dt.tzinfo | ||
|
|
||
|
|
||
| # Copyright (c) Django Software Foundation and individual contributors. | ||
|
|
@@ -42,7 +44,7 @@ def get_time_zone(time_zone_str: str) -> Optional[dt.tzinfo]: | |
| """ | ||
| try: | ||
| return pytz.timezone(time_zone_str) | ||
| except pytz.exceptions.UnknownTimeZoneError: | ||
| except pytzexceptions.UnknownTimeZoneError: | ||
| return None | ||
|
|
||
|
|
||
|
|
@@ -64,7 +66,7 @@ def as_utc(dattim: dt.datetime) -> dt.datetime: | |
| if dattim.tzinfo == UTC: | ||
| return dattim | ||
| elif dattim.tzinfo is None: | ||
| dattim = DEFAULT_TIME_ZONE.localize(dattim) | ||
| dattim = DEFAULT_TIME_ZONE.localize(dattim) # type: ignore | ||
|
|
||
| return dattim.astimezone(UTC) | ||
|
|
||
|
|
@@ -92,7 +94,7 @@ def as_local(dattim: dt.datetime) -> dt.datetime: | |
|
|
||
| def utc_from_timestamp(timestamp: float) -> dt.datetime: | ||
| """Return a UTC time from a timestamp.""" | ||
| return dt.datetime.utcfromtimestamp(timestamp).replace(tzinfo=UTC) | ||
| return UTC.localize(dt.datetime.utcfromtimestamp(timestamp)) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this the same? Is a non timezone aware date that is being localized treated as having been UTC time? Or would it treat it as local time?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here is the implementation of UTC.localize |
||
|
|
||
|
|
||
| def start_of_local_day(dt_or_d: | ||
|
|
@@ -102,13 +104,14 @@ def start_of_local_day(dt_or_d: | |
| date = now().date() # type: dt.date | ||
| elif isinstance(dt_or_d, dt.datetime): | ||
| date = dt_or_d.date() | ||
| return DEFAULT_TIME_ZONE.localize(dt.datetime.combine(date, dt.time())) | ||
| return DEFAULT_TIME_ZONE.localize(dt.datetime.combine( # type: ignore | ||
| date, dt.time())) | ||
|
|
||
|
|
||
| # Copyright (c) Django Software Foundation and individual contributors. | ||
| # All rights reserved. | ||
| # https://github.com/django/django/blob/master/LICENSE | ||
| def parse_datetime(dt_str: str) -> dt.datetime: | ||
| def parse_datetime(dt_str: str) -> Optional[dt.datetime]: | ||
| """Parse a string and return a datetime.datetime. | ||
|
|
||
| This function supports time zone offsets. When the input contains one, | ||
|
|
@@ -134,14 +137,12 @@ def parse_datetime(dt_str: str) -> dt.datetime: | |
| if tzinfo_str[0] == '-': | ||
| offset = -offset | ||
| tzinfo = dt.timezone(offset) | ||
| else: | ||
| tzinfo = None | ||
| kws = {k: int(v) for k, v in kws.items() if v is not None} | ||
| kws['tzinfo'] = tzinfo | ||
| return dt.datetime(**kws) | ||
|
|
||
|
|
||
| def parse_date(dt_str: str) -> dt.date: | ||
| def parse_date(dt_str: str) -> Optional[dt.date]: | ||
| """Convert a date string to a date object.""" | ||
| try: | ||
| return dt.datetime.strptime(dt_str, DATE_STR_FORMAT).date() | ||
|
|
@@ -180,9 +181,8 @@ def get_age(date: dt.datetime) -> str: | |
| def formatn(number: int, unit: str) -> str: | ||
| """Add "unit" if it's plural.""" | ||
| if number == 1: | ||
| return "1 %s" % unit | ||
| elif number > 1: | ||
| return "%d %ss" % (number, unit) | ||
| return '1 {}'.format(unit) | ||
| return '{:d} {}s'.format(number, unit) | ||
|
|
||
| # pylint: disable=invalid-sequence-index | ||
| def q_n_r(first: int, second: int) -> Tuple[int, int]: | ||
|
|
@@ -211,4 +211,4 @@ def q_n_r(first: int, second: int) -> Tuple[int, int]: | |
| if minute > 0: | ||
| return formatn(minute, 'minute') | ||
|
|
||
| return formatn(second, 'second') if second > 0 else "0 seconds" | ||
| return formatn(second, 'second') | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,10 +8,8 @@ | |
|
|
||
| _LOGGER = logging.getLogger(__name__) | ||
|
|
||
| _UNDEFINED = object() | ||
|
|
||
|
|
||
| def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \ | ||
| def load_json(filename: str, default: Union[List, Dict, None] = None) \ | ||
| -> Union[List, Dict]: | ||
| """Load JSON data from a file and return as dict or list. | ||
|
|
||
|
|
@@ -29,7 +27,7 @@ def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \ | |
| except OSError as error: | ||
| _LOGGER.exception('JSON file reading failed: %s', filename) | ||
| raise HomeAssistantError(error) | ||
| return {} if default is _UNDEFINED else default | ||
| return {} if default is None else default | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This not the same. By removing the sentinel value, it is no longer possible to have
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There were no uses of
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was no use yet, as the storage helper is very new. I don't think that we should change this.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The storage helper was the lone use. I meant that no one else used By using sentinel value the function becomes not type-safe, as it accepts Another option is to have
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't feel too strongly I guess. Any place that was using
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually, you could also make the sentinal value an empty dict, that way you can keep it all as dict. Comparison is still done with
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It will generate a warning that the default value is mutable. Lets keep it |
||
|
|
||
|
|
||
| def save_json(filename: str, data: Union[List, Dict]): | ||
|
|
@@ -38,9 +36,9 @@ def save_json(filename: str, data: Union[List, Dict]): | |
| Returns True on success. | ||
| """ | ||
| try: | ||
| data = json.dumps(data, sort_keys=True, indent=4) | ||
| json_data = json.dumps(data, sort_keys=True, indent=4) | ||
| with open(filename, 'w', encoding='utf-8') as fdesc: | ||
| fdesc.write(data) | ||
| fdesc.write(json_data) | ||
| return True | ||
| except TypeError as error: | ||
| _LOGGER.exception('Failed to serialize to JSON: %s', | ||
|
|
@@ -50,4 +48,3 @@ def save_json(filename: str, data: Union[List, Dict]): | |
| _LOGGER.exception('Saving JSON file failed: %s', | ||
| filename) | ||
| raise HomeAssistantError(error) | ||
| return False | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -57,7 +57,7 @@ def compose_node(self, parent: yaml.nodes.Node, index) -> yaml.nodes.Node: | |
| last_line = self.line # type: int | ||
| node = super(SafeLineLoader, | ||
| self).compose_node(parent, index) # type: yaml.nodes.Node | ||
| node.__line__ = last_line + 1 | ||
| node.__line__ = last_line + 1 # type: ignore | ||
| return node | ||
|
|
||
|
|
||
|
|
@@ -69,7 +69,7 @@ def load_yaml(fname: str) -> Union[List, Dict]: | |
| # We convert that to an empty dict | ||
| return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict() | ||
| except yaml.YAMLError as exc: | ||
| _LOGGER.error(exc) | ||
| _LOGGER.error(str(exc)) | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exceptions have a
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. mypy complains that it is expecting str and got an Exception. Rather than add
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think mypy is wrong in this case. I'd prefer the ignore, but you decide, 🤷♂️.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think I am leaning towards |
||
| raise HomeAssistantError(exc) | ||
| except UnicodeDecodeError as exc: | ||
| _LOGGER.error("Unable to read file %s: %s", fname, exc) | ||
|
|
@@ -232,6 +232,8 @@ def _load_secret_yaml(secret_path: str) -> Dict: | |
| _LOGGER.debug('Loading %s', secret_path) | ||
| try: | ||
| secrets = load_yaml(secret_path) | ||
| if not isinstance(secrets, dict): | ||
| raise HomeAssistantError('Secrets is not a dictionary') | ||
| if 'logger' in secrets: | ||
| logger = str(secrets['logger']).lower() | ||
| if logger == 'debug': | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Don't wrap it into a slow coroutine.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Any suggestions how to properly do that?
If the caller knows that the target is a @callback and is not afraid of exceptions - they can call it directly.
If the caller knows that the target is a @callback, they are afraid of exceptions and don't care about return value - they can call
loop.call_soonmanually (i.e. the previous implementation)If the caller knows that the target is a @callback, they are afraid of exceptions and they want the return value - the old implementation didn't provide that, the new one does.
If the caller doesn't know what
targetis and they need the return value - same as (3)If the caller doesn't know what
targetis and they don't need the return value - I agree that the old implementation is better, but it would be also better for coro not to be wrapped in task for such a case.Generally having a single function with different return types is bad.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For typing, yes but not for python itself
This change is very bad for the core and extend the memory footprint. We should hold the core slim and don't blow it up for a not exists type checking. Python is not strict and I hope that will be change in 4.x like c++ but actually I see no reason to make this bad change. In that case, I dislike this.
And no, user should use our internal job handler or events stuff and not use the raw asyncio functions.
And the real solution is to remove
@callbackand make all that coreroutine. But it is good like it is now.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So I would say that it's kinda a miracle that not returning anything has never caused any issues. The majority of the callbacks in use are actually being used inside
helpers/event.py. There we callhass.async_run_jobwhich will run callbacks right away and enqueue the other types of functions.Let's measure how much overhead this adds with our benchmark module.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
On
devWith this PR:
That's just a quick check of running it a couple of times. That's a significant slowdown.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
(this converted both the listener inside the benchmark as the function defined inside async_track_state_change to async functions)
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Interestingly, if I just convert the function inside track_state_change to be async, I get back to ~5.5 seconds per run
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think a proper solution is to make 5 new (type-safe) functions for all the usecases I mentioned above, then convert the callers and then delete the functions that are not actually used
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I agree with that approach.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hmm. That is python and not type safe. To make the code complex for nothing is not realy cool. If that is a c++ project, I would agree but not with python...