Skip to content

Commit

Permalink
Fix PermissionError issue on windows systems (#111)
Browse files Browse the repository at this point in the history
* add changelog

* add _run_safely method and context manager

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add pr to changelog entry

* add typing

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
northernSage and pre-commit-ci[bot] authored Jan 15, 2022
1 parent 818a6a9 commit 6ae8e0e
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 7 deletions.
4 changes: 3 additions & 1 deletion CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ Version 0.6.0
Unreleased

- A custom ``hash_method`` may now be provided to ``FileSystemCache`` for
hashing keys.
hashing keys. :pr:`107`

- Fix ``PermissionError`` issue with ``FileSystemCache`` on Windows. :pr:111


Version 0.5.0
Expand Down
50 changes: 44 additions & 6 deletions src/cachelib/file.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import errno
import logging
import os
import platform
import tempfile
import typing as _t
from contextlib import contextmanager
from hashlib import md5
from pathlib import Path
from time import sleep
from time import time

from cachelib.base import BaseCache
Expand Down Expand Up @@ -101,7 +104,7 @@ def _over_threshold(self) -> bool:
def _remove_expired(self, now: float) -> None:
for fname in self._list_dir():
try:
with open(fname, "rb") as f:
with self._safe_stream_open(fname, "rb") as f:
expires = self.serializer.load(f)
if expires != 0 and expires < now:
os.remove(fname)
Expand All @@ -119,7 +122,7 @@ def _remove_older(self) -> bool:
exp_fname_tuples = []
for fname in self._list_dir():
try:
with open(fname, "rb") as f:
with self._safe_stream_open(fname, "rb") as f:
exp_fname_tuples.append((self.serializer.load(f), fname))
except FileNotFoundError:
pass
Expand Down Expand Up @@ -186,7 +189,7 @@ def _get_filename(self, key: str) -> str:
def get(self, key: str) -> _t.Any:
filename = self._get_filename(key)
try:
with open(filename, "rb") as f:
with self._safe_stream_open(filename, "rb") as f:
pickle_time = self.serializer.load(f)
if pickle_time == 0 or pickle_time >= time():
return self.serializer.load(f)
Expand Down Expand Up @@ -231,8 +234,10 @@ def set(
with os.fdopen(fd, "wb") as f:
self.serializer.dump(timeout, f) # this returns bool
self.serializer.dump(value, f)
os.replace(tmp, filename)
os.chmod(filename, self._mode)

self._run_safely(os.replace, tmp, filename)
self._run_safely(os.chmod, filename, self._mode)

fsize = Path(filename).stat().st_size
except OSError:
logging.warning(
Expand Down Expand Up @@ -264,7 +269,7 @@ def delete(self, key: str, mgmt_element: bool = False) -> bool:
def has(self, key: str) -> bool:
filename = self._get_filename(key)
try:
with open(filename, "rb") as f:
with self._safe_stream_open(filename, "rb") as f:
pickle_time = self.serializer.load(f)
if pickle_time == 0 or pickle_time >= time():
return True
Expand All @@ -279,3 +284,36 @@ def has(self, key: str) -> bool:
exc_info=True,
)
return False

def _run_safely(self, fn: _t.Callable, *args: _t.Any, **kwargs: _t.Any) -> _t.Any:
"""On Windows os.replace, os.chmod and open can yield
permission errors if executed by two different processes."""
if platform.system() == "Windows":
output = None
wait_step = 0.001
max_sleep_time = 10.0
total_sleep_time = 0.0

while total_sleep_time < max_sleep_time:
try:
output = fn(*args, **kwargs)
except PermissionError:
sleep(wait_step)
total_sleep_time += wait_step
wait_step *= 2
else:
break
else:
output = fn(*args, **kwargs)

return output

@contextmanager
def _safe_stream_open(self, path: str, mode: str) -> _t.Generator:
fs = self._run_safely(open, path, mode)
if fs is None:
raise OSError
try:
yield fs
finally:
fs.close()

0 comments on commit 6ae8e0e

Please sign in to comment.