Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions fileformats/core/fileset.py
Original file line number Diff line number Diff line change
Expand Up @@ -949,30 +949,6 @@ def hash_files(
file_hashes[str(path)] = crypto_obj.hexdigest()
return file_hashes

def __bytes_repr__(
self, cache: ty.Dict[ty.Any, str] # pylint: disable=unused-argument
) -> ty.Iterable[bytes]:
"""Provided for compatibility with Pydra's hashing function, return the contents
of all the files in the file-set in chunks

Parameters
----------
cache : dict[Any, str]
an object passed around by Pydra's hashing function to store cached versions
of previously hashed objects, to allow recursive structures

Yields
------
bytes
a chunk of bytes of length FILE_CHUNK_LEN_DEFAULT from the contents of all
files in the file-set.
"""
cls = type(self)
yield f"{cls.__module__}.{cls.__name__}:".encode()
for key, chunk_iter in self.byte_chunks():
yield (",'" + key + "'=").encode()
yield from chunk_iter

@classmethod
def referenced_types(cls) -> ty.Set[ty.Type[Classifier]]:
"""Returns a flattened list of nested types referenced within the fileset type
Expand Down
63 changes: 61 additions & 2 deletions fileformats/core/tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,14 @@
import time
import typing as ty
import pytest
from fileformats.core import FileSet, validated_property
from fileformats.generic import File, BinaryFile, Directory, FsObject
from fileformats.generic import File, BinaryFile, Directory, FsObject, SetOf
from fileformats.core import (
FileSet,
MockMixin,
validated_property,
mtime_cached_property,
)
from fileformats.text import TextFile
from fileformats.core.mixin import WithSeparateHeader
from fileformats.core.exceptions import UnsatisfiableCopyModeError
from conftest import write_test_file
Expand Down Expand Up @@ -54,6 +60,11 @@ def fsobject(luigi_file, bowser_dir, request):
assert False


@pytest.fixture
def mock_fileset():
return SetOf[TextFile].mock("/path/to/a/mock", "/path/to/another/mock")


@pytest.fixture
def dest_dir(work_dir):
dest_dir = work_dir / "new-dir"
Expand Down Expand Up @@ -523,3 +534,51 @@ def test_hash_files(fsobject: FsObject, work_dir: Path, dest_dir: Path):
)
cpy = fsobject.copy(dest_dir)
assert cpy.hash_files() == fsobject.hash_files()


class MtimeTestFile(File):

flag: int

@mtime_cached_property
def cached_prop(self):
return self.flag


def test_mtime_cached_property(tmp_path: Path):
fspath = tmp_path / "file_1.txt"
fspath.write_text("hello")

file = MtimeTestFile(fspath)

file.flag = 0
assert file.cached_prop == 0
# Need a long delay to ensure the mtime changes on Ubuntu and particularly on Windows
# On MacOS, the mtime resolution is much higher so not usually an issue. Use
# explicitly cache clearing if needed
time.sleep(2)
file.flag = 1
assert file.cached_prop == 0
time.sleep(2)
fspath.write_text("world")
assert file.cached_prop == 1


def test_mtime_cached_property_force_clear(tmp_path: Path):
fspath = tmp_path / "file_1.txt"
fspath.write_text("hello")

file = MtimeTestFile(fspath)

file.flag = 0
assert file.cached_prop == 0
file.flag = 1
MtimeTestFile.cached_prop.clear(file)
assert file.cached_prop == 1


def test_hash_mock_files(mock_fileset: MockMixin, work_dir: Path, dest_dir: Path):
file_hashes = mock_fileset.hash_files(relative_to="")
assert sorted(Path(p) for p in file_hashes) == sorted(
p for p in mock_fileset.fspaths
)
Loading