From a2caa9e15ec4346b47958889c55b9a26308a7cab Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Sat, 29 May 2021 14:01:42 -0400 Subject: [PATCH 01/54] add fiftyone module availability --- flash/core/utilities/imports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flash/core/utilities/imports.py b/flash/core/utilities/imports.py index 6be32e5842..3bc86b8b1b 100644 --- a/flash/core/utilities/imports.py +++ b/flash/core/utilities/imports.py @@ -75,6 +75,7 @@ def _compare_version(package: str, op, version) -> bool: _MATPLOTLIB_AVAILABLE = _module_available("matplotlib") _TRANSFORMERS_AVAILABLE = _module_available("transformers") _PYSTICHE_AVAILABLE = _module_available("pystiche") +_FIFTYONE_AVAILABLE = _module_available("fiftyone") if Version: _TORCHVISION_GREATER_EQUAL_0_9 = _compare_version("torchvision", operator.ge, "0.9.0") From 9e7b38e0f4e76abda45f113a613ba5265e6f79d5 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Sat, 29 May 2021 14:08:18 -0400 Subject: [PATCH 02/54] add fiftyone datasource --- flash/core/data/data_source.py | 15 +++++++++++++++ flash/video/classification/data.py | 19 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 0858e9a26a..287ffa7dab 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -42,6 +42,11 @@ from flash.core.data.properties import ProcessState, Properties from flash.core.data.utils import CurrentRunningStageFuncContext +if _FIFTYONE_AVAILABLE: + from fiftyone.core.collections import SampleCollection +else: + SampleCollection = None + # Credit to the PyTorchVision Team: # https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py#L10 @@ -461,3 +466,13 @@ class TensorDataSource(SequenceDataSource[torch.Tensor]): class NumpyDataSource(SequenceDataSource[np.ndarray]): """The ``NumpyDataSource`` is a ``SequenceDataSource`` which expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence of ``np.ndarray`` objects.""" + +class FiftyOneDataSource(SequenceDataSource[SampleCollection]): + """The ``FiftyOneDataSource`` is a ``SequenceDataSource`` which expects the input to + :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence + of FiftyOne Dataset objects.""" + + def predict_load_data(self, + data: SampleCollection, + dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + return [{DefaultDataKeys.INPUT: s.filepath} for s in data] diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 11c40bdc6c..5657b514ac 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -20,10 +20,25 @@ from torch.utils.data import RandomSampler, Sampler from flash.core.data.data_module import DataModule -from flash.core.data.data_source import DefaultDataKeys, DefaultDataSources, LabelsState, PathsDataSource +from flash.core.data.data_source import ( + DefaultDataKeys, + DefaultDataSources, + FiftyOneDataSource, + LabelsState, + PathsDataSource, +) from flash.core.data.process import Preprocess from flash.core.data.transforms import merge_transforms -from flash.core.utilities.imports import _KORNIA_AVAILABLE, _PYTORCHVIDEO_AVAILABLE +from flash.core.utilities.imports import ( + _KORNIA_AVAILABLE, + _FIFTYONE_AVAILABLE, + _PYTORCHVIDEO_AVAILABLE, +) + +if _FIFTYONE_AVAILABLE: + from fiftyone.core.collections import SampleCollection +else: + SampleCollection = None if _KORNIA_AVAILABLE: import kornia.augmentation as K From 2d55c9e69013a9e18a9d11eae0e55df9ddd8136e Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Sat, 29 May 2021 14:38:17 -0400 Subject: [PATCH 03/54] add video classification data source --- flash/core/data/data_module.py | 80 +++++++++++++++++++++++++++ flash/core/data/data_source.py | 2 + flash/video/classification/data.py | 89 +++++++++++++++++++++++++++++- 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 58a3337e1d..e60d57fbbd 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -30,6 +30,12 @@ from flash.core.data.data_source import DatasetDataSource, DataSource, DefaultDataSources from flash.core.data.splits import SplitDataset from flash.core.data.utils import _STAGES_PREFIX +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE + +if _FIFTYONE_AVAILABLE: + from fiftyone.core.collections import SampleCollection +else: + SampleCollection = None class DataModule(pl.LightningDataModule): @@ -1024,3 +1030,77 @@ def from_datasets( num_workers=num_workers, **preprocess_kwargs, ) + + @classmethod + def from_fiftyone( + cls, + train_dataset: Optional[SampleCollection] = None, + val_dataset: Optional[SampleCollection] = None, + test_dataset: Optional[SampleCollection] = None, + predict_dataset: Optional[SampleCollection] = None, + train_transform: Optional[Dict[str, Callable]] = None, + val_transform: Optional[Dict[str, Callable]] = None, + test_transform: Optional[Dict[str, Callable]] = None, + predict_transform: Optional[Dict[str, Callable]] = None, + data_fetcher: Optional[BaseDataFetcher] = None, + preprocess: Optional[Preprocess] = None, + val_split: Optional[float] = None, + batch_size: int = 4, + num_workers: Optional[int] = None, + **preprocess_kwargs: Any, + ) -> 'DataModule': + """Creates a :class:`~flash.core.data.data_module.DataModule` object + from the given FiftyOne Datasets using the + :class:`~flash.core.data.data_source.DataSource` of name + :attr:`~flash.core.data.data_source.DefaultDataSources.FIFTYONE` + from the passed or constructed :class:`~flash.core.data.process.Preprocess`. + Args: + train_dataset: The FiftyOne Dataset containing the train data. + val_dataset: The FiftyOne Dataset containing the validation data. + test_dataset: The FiftyOne Dataset containing the test data. + predict_dataset: The FiftyOne Dataset containing the predict data. + train_transform: The dictionary of transforms to use during training which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + val_transform: The dictionary of transforms to use during validation which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + test_transform: The dictionary of transforms to use during testing which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + predict_transform: The dictionary of transforms to use during predicting which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + data_fetcher: The :class:`~flash.core.data.callback.BaseDataFetcher` to pass to the + :class:`~flash.core.data.data_module.DataModule`. + preprocess: The :class:`~flash.core.data.data.Preprocess` to pass to the + :class:`~flash.core.data.data_module.DataModule`. If ``None``, ``cls.preprocess_cls`` + will be constructed and used. + val_split: The ``val_split`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + batch_size: The ``batch_size`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + num_workers: The ``num_workers`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + preprocess_kwargs: Additional keyword arguments to use when constructing the preprocess. Will only be used + if ``preprocess = None``. + Returns: + The constructed data module. + Examples:: + data_module = DataModule.from_folders( + train_folder="train_folder", + train_transform={ + "to_tensor_transform": torch.as_tensor, + }, + ) + """ + return cls.from_data_source( + DefaultDataSources.FIFTYONE, + train_dataset, + val_dataset, + test_dataset, + predict_dataset, + train_transform=train_transform, + val_transform=val_transform, + test_transform=test_transform, + predict_transform=predict_transform, + data_fetcher=data_fetcher, + preprocess=preprocess, + val_split=val_split, + batch_size=batch_size, + num_workers=num_workers, + **preprocess_kwargs, + ) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 287ffa7dab..0bcc12a21f 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -41,6 +41,7 @@ from flash.core.data.auto_dataset import AutoDataset, BaseAutoDataset, IterableAutoDataset from flash.core.data.properties import ProcessState, Properties from flash.core.data.utils import CurrentRunningStageFuncContext +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: from fiftyone.core.collections import SampleCollection @@ -150,6 +151,7 @@ class DefaultDataSources(LightningEnum): CSV = "csv" JSON = "json" DATASET = "dataset" + FIFTYONE = "fiftyone" # TODO: Create a FlashEnum class??? def __hash__(self) -> int: diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 5657b514ac..5ef69489b9 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -30,8 +30,8 @@ from flash.core.data.process import Preprocess from flash.core.data.transforms import merge_transforms from flash.core.utilities.imports import ( - _KORNIA_AVAILABLE, _FIFTYONE_AVAILABLE, + _KORNIA_AVAILABLE, _PYTORCHVIDEO_AVAILABLE, ) @@ -47,6 +47,7 @@ from pytorchvideo.data.clip_sampling import ClipSampler, make_clip_sampler from pytorchvideo.data.encoded_video import EncodedVideo from pytorchvideo.data.encoded_video_dataset import EncodedVideoDataset, labeled_encoded_video_dataset + from pytorchvideo.data.labeled_video_paths import LabeledVideoPaths from pytorchvideo.transforms import ( ApplyTransformToKey, RandomShortSideScale, @@ -126,6 +127,84 @@ def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) +class VideoClassificationFiftyOneDataSource(FiftyOneDataSource): + + def __init__( + self, + clip_sampler: 'ClipSampler', + label_field: str = "ground_truth", + video_sampler: Type[Sampler] = torch.utils.data.RandomSampler, + decode_audio: bool = True, + decoder: str = "pyav", + ): + super().__init__() + self.label_field = label_field + self.clip_sampler = clip_sampler + self.video_sampler = video_sampler + self.decode_audio = decode_audio + self.decoder = decoder + + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': + label_to_class_mapping = dict(list(enumerate(data.default_classes))) + class_to_label_mapping = {c:l for l,c in label_to_class_mapping.items()} + + labeled_video_paths = [] + for s in data: + label = class_to_label_mapping[s[self.label_field].label] + filepath = s.filepath + labeled_video_paths.append((filepath,label)) + + labeled_video_paths = LabeledVideoPaths( + labeled_video_paths + ) + + ds: EncodedVideoDataset = EncodedVideoDataset( + labeled_video_paths, + self.clip_sampler, + video_sampler=self.video_sampler, + decode_audio=self.decode_audio, + decoder=self.decoder, + ) + if self.training: + self.set_state(LabelsState(label_to_class_mapping)) + dataset.num_classes = len(class_to_label_mapping) + return ds + + def _encoded_video_to_dict(self, video) -> Dict[str, Any]: + ( + clip_start, + clip_end, + clip_index, + aug_index, + is_last_clip, + ) = self.clip_sampler(0.0, video.duration) + + loaded_clip = video.get_clip(clip_start, clip_end) + + clip_is_null = ( + loaded_clip is None or loaded_clip["video"] is None or (loaded_clip["audio"] is None and self.decode_audio) + ) + if clip_is_null: + raise MisconfigurationException( + f"The provided video is too short {video.duration} to be clipped at {self.clip_sampler._clip_duration}" + ) + frames = loaded_clip["video"] + audio_samples = loaded_clip["audio"] + return { + "video": frames, + "video_name": video.name, + "video_index": 0, + "clip_index": clip_index, + "aug_index": aug_index, + **({ + "audio": audio_samples + } if audio_samples is not None else {}), + } + + def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: + return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) + + class VideoClassificationPreprocess(Preprocess): def __init__( @@ -134,6 +213,7 @@ def __init__( val_transform: Optional[Dict[str, Callable]] = None, test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, + label_field: str = "ground_truth", clip_sampler: Union[str, 'ClipSampler'] = "random", clip_duration: float = 2, clip_sampler_kwargs: Dict[str, Any] = None, @@ -179,6 +259,13 @@ def __init__( decode_audio=decode_audio, decoder=decoder, ), + DefaultDataSources.FIFTYONE: VideoClassificationFiftyOneDataSource( + clip_sampler, + label_field=label_field, + video_sampler=video_sampler, + decode_audio=decode_audio, + decoder=decoder, + ), }, default_data_source=DefaultDataSources.FILES, ) From 35ef87e7aef958eeb437a98ffe48361478f3c0f5 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Sat, 29 May 2021 15:19:23 -0400 Subject: [PATCH 04/54] add fiftyone classification serializer --- flash/core/classification.py | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/flash/core/classification.py b/flash/core/classification.py index 7926e20fb6..6488451c5c 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -21,6 +21,10 @@ from flash.core.data.data_source import LabelsState from flash.core.data.process import Serializer from flash.core.model import Task +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE + +if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Classification, Classifications def binary_cross_entropy_with_logits(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: @@ -158,3 +162,69 @@ def serialize(self, sample: Any) -> Union[int, List[int], str, List[str]]: else: rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) return classes + + +class FiftyOneLabels(Classes): + """A :class:`.Serializer` which converts the model outputs (assumed to be logits) to the label of the + argmax classification all stored as a FiftyOne Classification label. + Args: + labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not + provided, will attempt to get them from the :class:`.LabelsState`. + multi_label: If true, treats outputs as multi label logits and creates + FiftyOne Classifications. + threshold: The threshold to use for multi_label classification. + """ + + def __init__(self, labels: Optional[List[str]] = None, multi_label: bool = False, threshold: float = 0.5): + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + + super().__init__(multi_label=multi_label, threshold=threshold) + self._labels = labels + + if labels is not None: + self.set_state(LabelsState(labels)) + + def serialize(self, sample: Any) -> Union[Classification, Classifications]: + labels = None + + if self._labels is not None: + labels = self._labels + else: + state = self.get_state(LabelsState) + if state is not None: + labels = state.labels + + classes = super().serialize(sample) + + if self.multi_label: + probabilities = torch.sigmoid(sample).tolist() + else: + probabilities = torch.softmax(sample, -1).tolist() + + if labels is not None: + if self.multi_label: + classifications = [] + for probs, cls in zip(probabilities, classes): + fo_cls = Classification( + label = labels[cls], + confidence = max(prob), + ) + classifications.append(fo_cls) + fo_labels = Classifications( + classifications=classifications, + ) + else: + fo_labels = Classification( + label = labels[classes], + confidence = max(probabilities), + logits = sample.tolist(), + ) + else: + rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) + fo_labels = Classification( + label = classes, + confidence = max(probabilities), + ) + + return fo_labels From 2c94a5c68b9445586cbb95f45b5b00a5fb91e4b1 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Wed, 2 Jun 2021 11:18:26 -0400 Subject: [PATCH 05/54] optimizations, rework fo serializer --- flash/core/classification.py | 40 ++++++++++++----- flash/core/data/data_module.py | 12 +++++- flash/core/data/data_source.py | 9 ++-- flash/video/classification/data.py | 69 +++++++++--------------------- 4 files changed, 64 insertions(+), 66 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 6488451c5c..d8c30ff0bf 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -73,11 +73,11 @@ class ClassificationSerializer(Serializer): def __init__(self, multi_label: bool = False): super().__init__() - self._mutli_label = multi_label + self._multi_label = multi_label @property def multi_label(self) -> bool: - return self._mutli_label + return self._multi_label class Logits(ClassificationSerializer): @@ -164,7 +164,7 @@ def serialize(self, sample: Any) -> Union[int, List[int], str, List[str]]: return classes -class FiftyOneLabels(Classes): +class FiftyOneLabels(ClassificationSerializer): """A :class:`.Serializer` which converts the model outputs (assumed to be logits) to the label of the argmax classification all stored as a FiftyOne Classification label. Args: @@ -173,13 +173,23 @@ class FiftyOneLabels(Classes): multi_label: If true, treats outputs as multi label logits and creates FiftyOne Classifications. threshold: The threshold to use for multi_label classification. + store_logits: Boolean determining whether to store logits in + the FiftyOne labels """ - def __init__(self, labels: Optional[List[str]] = None, multi_label: bool = False, threshold: float = 0.5): + def __init__( + self, + labels: Optional[List[str]] = None, + multi_label: bool = False, + threshold: float = 0.5, + store_logits: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") - super().__init__(multi_label=multi_label, threshold=threshold) + super().__init__(multi_label=multi_label) + self.threshold = threshold + self.store_logits = store_logits self._labels = labels if labels is not None: @@ -195,36 +205,46 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: if state is not None: labels = state.labels - classes = super().serialize(sample) + logits = None + if self.store_logits: + logits = sample.tolist() if self.multi_label: + one_hot = (sample.sigmoid() > self.threshold).int().tolist() + classes = [] + for index, value in enumerate(one_hot): + if value == 1: + classes.append(index) probabilities = torch.sigmoid(sample).tolist() else: + classes = torch.argmax(sample, -1).tolist() probabilities = torch.softmax(sample, -1).tolist() if labels is not None: if self.multi_label: classifications = [] - for probs, cls in zip(probabilities, classes): + for cls in classes: fo_cls = Classification( label = labels[cls], - confidence = max(prob), + confidence = probabilities[cls], ) classifications.append(fo_cls) fo_labels = Classifications( - classifications=classifications, + classifications = classifications, + logits = logits, ) else: fo_labels = Classification( label = labels[classes], confidence = max(probabilities), - logits = sample.tolist(), + logits = logits, ) else: rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) fo_labels = Classification( label = classes, confidence = max(probabilities), + logits = logits ) return fo_labels diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index e60d57fbbd..34a46b8d72 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -1054,6 +1054,7 @@ def from_fiftyone( :class:`~flash.core.data.data_source.DataSource` of name :attr:`~flash.core.data.data_source.DefaultDataSources.FIFTYONE` from the passed or constructed :class:`~flash.core.data.process.Preprocess`. + Args: train_dataset: The FiftyOne Dataset containing the train data. val_dataset: The FiftyOne Dataset containing the validation data. @@ -1077,11 +1078,18 @@ def from_fiftyone( num_workers: The ``num_workers`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. preprocess_kwargs: Additional keyword arguments to use when constructing the preprocess. Will only be used if ``preprocess = None``. + Returns: The constructed data module. + Examples:: - data_module = DataModule.from_folders( - train_folder="train_folder", + + train_dataset = fo.Dataset.from_dir( + "/path/to/dataset", + dataset_type=fo.types.ImageClassificationDirectoryTree, + ) + data_module = DataModule.from_fiftyone( + train_data = train_dataset, train_transform={ "to_tensor_transform": torch.as_tensor, }, diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 0bcc12a21f..1f1eceb598 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -469,12 +469,11 @@ class NumpyDataSource(SequenceDataSource[np.ndarray]): """The ``NumpyDataSource`` is a ``SequenceDataSource`` which expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence of ``np.ndarray`` objects.""" -class FiftyOneDataSource(SequenceDataSource[SampleCollection]): - """The ``FiftyOneDataSource`` is a ``SequenceDataSource`` which expects the input to - :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence - of FiftyOne Dataset objects.""" +class FiftyOneDataSource(DataSource[SampleCollection]): + """The ``FiftyOneDataSource`` expects the input to + :meth:`~flash.core.data.data_source.DataSource.load_data` to be FiftyOne Dataset objects.""" def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: s.filepath} for s in data] + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 5ef69489b9..4c5350af7d 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -127,32 +127,33 @@ def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) -class VideoClassificationFiftyOneDataSource(FiftyOneDataSource): +class VideoClassificationFiftyOneDataSource(FiftyOneDataSource, VideoClassificationPathsDataSource): def __init__( self, clip_sampler: 'ClipSampler', - label_field: str = "ground_truth", video_sampler: Type[Sampler] = torch.utils.data.RandomSampler, decode_audio: bool = True, decoder: str = "pyav", + label_field: str = "ground_truth", ): - super().__init__() + super().__init__( + clip_sampler=clip_sampler, + video_sampler=video_sampler, + decode_audio=decode_audio, + decoder=decoder, + ) self.label_field = label_field - self.clip_sampler = clip_sampler - self.video_sampler = video_sampler - self.decode_audio = decode_audio - self.decoder = decoder def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': - label_to_class_mapping = dict(list(enumerate(data.default_classes))) + label_to_class_mapping = dict(enumerate(data.default_classes)) class_to_label_mapping = {c:l for l,c in label_to_class_mapping.items()} - labeled_video_paths = [] - for s in data: - label = class_to_label_mapping[s[self.label_field].label] - filepath = s.filepath - labeled_video_paths.append((filepath,label)) + filepaths = data.values("filepath") + labels = data.values(self.label_field + ".label") + + targets = [class_to_label_mapping[l] for l in labels] + labeled_video_paths = list(zip(filepaths, targets)) labeled_video_paths = LabeledVideoPaths( labeled_video_paths @@ -170,40 +171,6 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'E dataset.num_classes = len(class_to_label_mapping) return ds - def _encoded_video_to_dict(self, video) -> Dict[str, Any]: - ( - clip_start, - clip_end, - clip_index, - aug_index, - is_last_clip, - ) = self.clip_sampler(0.0, video.duration) - - loaded_clip = video.get_clip(clip_start, clip_end) - - clip_is_null = ( - loaded_clip is None or loaded_clip["video"] is None or (loaded_clip["audio"] is None and self.decode_audio) - ) - if clip_is_null: - raise MisconfigurationException( - f"The provided video is too short {video.duration} to be clipped at {self.clip_sampler._clip_duration}" - ) - frames = loaded_clip["video"] - audio_samples = loaded_clip["audio"] - return { - "video": frames, - "video_name": video.name, - "video_index": 0, - "clip_index": clip_index, - "aug_index": aug_index, - **({ - "audio": audio_samples - } if audio_samples is not None else {}), - } - - def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) - class VideoClassificationPreprocess(Preprocess): @@ -213,14 +180,18 @@ def __init__( val_transform: Optional[Dict[str, Callable]] = None, test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, - label_field: str = "ground_truth", clip_sampler: Union[str, 'ClipSampler'] = "random", clip_duration: float = 2, clip_sampler_kwargs: Dict[str, Any] = None, video_sampler: Type[Sampler] = torch.utils.data.RandomSampler, decode_audio: bool = True, decoder: str = "pyav", + **data_source_kwargs, ): + """ + ``data_source_kwargs`` are source-specific keyword arguments that are + passed to the ``DataSource`` constructors + """ self.clip_sampler = clip_sampler self.clip_duration = clip_duration self.clip_sampler_kwargs = clip_sampler_kwargs @@ -261,10 +232,10 @@ def __init__( ), DefaultDataSources.FIFTYONE: VideoClassificationFiftyOneDataSource( clip_sampler, - label_field=label_field, video_sampler=video_sampler, decode_audio=decode_audio, decoder=decoder, + **data_source_kwargs, ), }, default_data_source=DefaultDataSources.FILES, From c6587ea4ccc74854c9c191b4e8d1958d59970378 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Wed, 2 Jun 2021 21:42:46 -0400 Subject: [PATCH 06/54] support classification, detection, segmentation --- flash/core/classification.py | 2 +- flash/core/data/data_source.py | 34 ++++++++ flash/image/classification/data.py | 8 +- flash/image/data.py | 8 +- flash/image/detection/data.py | 99 ++++++++++++++++++++++- flash/image/detection/serialization.py | 73 +++++++++++++++++ flash/image/segmentation/data.py | 51 +++++++++++- flash/image/segmentation/serialization.py | 15 +++- flash/video/classification/data.py | 4 +- 9 files changed, 285 insertions(+), 9 deletions(-) create mode 100644 flash/image/detection/serialization.py diff --git a/flash/core/classification.py b/flash/core/classification.py index d8c30ff0bf..55cfec53bb 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -242,7 +242,7 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: else: rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) fo_labels = Classification( - label = classes, + label = str(classes), confidence = max(probabilities), logits = logits ) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 1f1eceb598..8276f682ff 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -473,6 +473,40 @@ class FiftyOneDataSource(DataSource[SampleCollection]): """The ``FiftyOneDataSource`` expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be FiftyOne Dataset objects.""" + def __init__(self, label_field: str = "ground_truth"): + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + super().__init__() + self.label_field = label_field + + def load_data(self, + data: SampleCollection, + dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + """Takes ``data``, a FiftyOne SampleCollection (generally a + ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and + parses sample filenames and labels from the given label field into a + list of inputs and targets. + """ + filepaths = data.values("filepath") + _, label_path = data._get_label_field_path(self.label_field, "label") + targets = data.values(label_path) + + classes = data.default_classes + if not classes: + classes = data.distinct(label_path) + + if dataset is not None: + dataset.num_classes = len(classes) + + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + to_idx = lambda t: class_to_idx[t] + if targets and isinstance(targets[0], list): + to_idx = lambda t: [class_to_idx[x] for x in t] + + + data = zip(filepaths, targets) + return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t)} for f,t in data] + def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: diff --git a/flash/image/classification/data.py b/flash/image/classification/data.py index ec7ee4a3a0..ec90e1e8dc 100644 --- a/flash/image/classification/data.py +++ b/flash/image/classification/data.py @@ -25,7 +25,7 @@ from flash.core.data.process import Preprocess from flash.core.utilities.imports import _IMAGE_AVAILABLE, _MATPLOTLIB_AVAILABLE from flash.image.classification.transforms import default_transforms, train_default_transforms -from flash.image.data import ImageNumpyDataSource, ImagePathsDataSource, ImageTensorDataSource +from flash.image.data import ImageFiftyOneDataSource, ImageNumpyDataSource, ImagePathsDataSource, ImageTensorDataSource if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -49,7 +49,12 @@ def __init__( test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, image_size: Tuple[int, int] = (196, 196), + **data_source_kwargs, ): + """ + ``data_source_kwargs`` are source-specific keyword arguments that are + passed to the ``DataSource`` constructors + """ self.image_size = image_size super().__init__( @@ -58,6 +63,7 @@ def __init__( test_transform=test_transform, predict_transform=predict_transform, data_sources={ + DefaultDataSources.FIFTYONE: ImageFiftyOneDataSource(**data_source_kwargs), DefaultDataSources.FILES: ImagePathsDataSource(), DefaultDataSources.FOLDERS: ImagePathsDataSource(), DefaultDataSources.NUMPY: ImageNumpyDataSource(), diff --git a/flash/image/data.py b/flash/image/data.py index 06cee5bf7f..d74689ddee 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -15,7 +15,7 @@ import torch -from flash.core.data.data_source import DefaultDataKeys, NumpyDataSource, PathsDataSource, TensorDataSource +from flash.core.data.data_source import DefaultDataKeys, FiftyOneDataSource, NumpyDataSource, PathsDataSource, TensorDataSource from flash.core.utilities.imports import _TORCHVISION_AVAILABLE if _TORCHVISION_AVAILABLE: @@ -47,3 +47,9 @@ class ImageNumpyDataSource(NumpyDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: sample[DefaultDataKeys.INPUT] = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) return sample + +class ImageFiftyOneDataSource(FiftyOneDataSource): + + def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: + sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + return sample diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 66cb43d4f0..eb64ecc7d7 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -17,14 +17,17 @@ from flash.core.data.callback import BaseDataFetcher from flash.core.data.data_module import DataModule from flash.core.data.data_source import DataSource, DefaultDataKeys, DefaultDataSources -from flash.core.data.process import Preprocess -from flash.core.utilities.imports import _COCO_AVAILABLE, _TORCHVISION_AVAILABLE -from flash.image.data import ImagePathsDataSource +from flash.core.data.process import Preprocess, Serializer +from flash.core.utilities.imports import _COCO_AVAILABLE, _FIFTYONE_AVAILABLE, _TORCHVISION_AVAILABLE +from flash.image.data import ImagePathsDataSource, ImageFiftyOneDataSource from flash.image.detection.transforms import default_transforms if _COCO_AVAILABLE: from pycocotools.coco import COCO +if _FIFTYONE_AVAILABLE: + from fiftyone.core.collections import SampleCollection + if _TORCHVISION_AVAILABLE: from torchvision.datasets.folder import default_loader @@ -90,6 +93,88 @@ def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return sample +class ObjectDetectionFiftyOneDataSource(ImageFiftyOneDataSource): + + def __init__(self, label_field: str = "ground_truth", iscrowd: str = "attributes.iscrowd.value"): + """Constructs an ObjectDetectionFiftyOneDataSource from a FiftyOne + SampleCollection using the given fields + + Args: + label_field: label field name containing information to construct + targets + iscrowd: name of the subfield of detections that stores iscrowd + information + """ + super().__init__(label_field=label_field) + self.iscrowd = iscrowd + + def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): + xmin *= img_w + ymin *= img_h + box_w *= img_w + box_h *= img_h + xmax = xmin + box_w + ymax = ymin + box_h + output_bbox = [xmin, ymin, xmax, ymax] + return output_bbox, box_w*box_h + + def load_data(self, + data: SampleCollection, + dataset: Optional[Any] = None) -> Sequence[Dict[str, Any]]: + """Takes ``data``, a FiftyOne SampleCollection (generally a + ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and + parses sample filenames and detections from the given label field into a + list of inputs and targets. + """ + data.compute_metadata() + filepaths = data.values("filepath") + labels = data.values(self.label_field + ".detections.label") + bboxes = data.values(self.label_field + ".detections.bounding_box") + iscrowds = data.values(self.label_field + ".detections." + self.iscrowd) + widths = data.values("metadata.width") + heights = data.values("metadata.height") + + classes = data.default_classes + if not classes: + classes = data.distinct(self.label_field + ".detections.label") + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + if dataset is not None: + dataset.num_classes = len(classes) + + data = zip(filepaths, widths, heights, labels, bboxes, iscrowds) + + output_data = [] + img_id = 1 + for fp, w, h, sample_labs, sample_boxes, sample_iscrowd in data: + output_boxes = [] + output_labs = [] + output_iscrowd = [] + output_areas = [] + for lab, box, iscrowd in zip(sample_labs, sample_boxes, sample_iscrowd): + output_box, output_area = self._reformat_bbox(box[0], box[1], box[2], box[3], w, h) + output_areas.append(output_area) + output_labs.append(class_to_idx[lab]) + output_boxes.append(output_box) + if iscrowd is None: + iscrowd = 0 + output_iscrowd.append(iscrowd) + output_data.append( + dict( + input=fp, + target=dict( + boxes=output_boxes, + labels=output_labs, + image_id=img_id, + area=output_areas, + iscrowd=output_iscrowd, + ) + ) + ) + img_id += 1 + + return output_data + + class ObjectDetectionPreprocess(Preprocess): def __init__( @@ -98,13 +183,21 @@ def __init__( val_transform: Optional[Dict[str, Callable]] = None, test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, + **data_source_kwargs, ): + """ + ``data_source_kwargs`` are source-specific keyword arguments that are + passed to the ``DataSource`` constructors + """ super().__init__( train_transform=train_transform, val_transform=val_transform, test_transform=test_transform, predict_transform=predict_transform, data_sources={ + DefaultDataSources.FIFTYONE: ObjectDetectionFiftyOneDataSource( + **data_source_kwargs + ), DefaultDataSources.FILES: ImagePathsDataSource(), DefaultDataSources.FOLDERS: ImagePathsDataSource(), "coco": COCODataSource(), diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py new file mode 100644 index 0000000000..9df4535b7d --- /dev/null +++ b/flash/image/detection/serialization.py @@ -0,0 +1,73 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, List, Optional + +from pytorch_lightning.utilities import rank_zero_warn + +from flash.core.data.data_source import LabelsState +from flash.core.data.process import Serializer +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE + +if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Detection, Detections + +class FiftyOneDetectionLabels(Serializer): + """A :class:`.Serializer` which converts the model outputs to a FiftyOne Detections label. + """ + + def __init__( + self, + labels: Optional[List[str]] = None, + ): + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + super().__init__() + self._labels = labels + + if labels is not None: + self.set_state(LabelsState(labels)) + + def serialize(self, sample: List[Dict[str, Any]]) -> Detections: + labels = None + + if self._labels is not None: + labels = self._labels + else: + state = self.get_state(LabelsState) + if state is not None: + labels = state.labels + else: + rank_zero_warn("No LabelsState was found, this serializer will return integer class labels.", UserWarning) + + detections = [] + + for det in sample: + xmin, ymin, xmax, ymax = [c.tolist() for c in det["boxes"]] + box = [xmin, ymin, xmax-xmin, ymax-ymin] + + label = det["labels"].tolist() + if labels is not None: + label = labels[label] + else: + label = str(int(label)) + + score = det["scores"].tolist() + detections.append(Detection( + label = label, + bounding_box = box, + confidence = score, + )) + + + return Detections(detections=detections) diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 4990fc59e9..0eb213777f 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -29,6 +29,7 @@ from flash.core.data.data_source import ( DefaultDataKeys, DefaultDataSources, + FiftyOneDataSource, ImageLabelsMap, NumpyDataSource, PathsDataSource, @@ -36,10 +37,13 @@ TensorDataSource, ) from flash.core.data.process import Preprocess -from flash.core.utilities.imports import _IMAGE_AVAILABLE, _MATPLOTLIB_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE, _MATPLOTLIB_AVAILABLE from flash.image.segmentation.serialization import SegmentationLabels from flash.image.segmentation.transforms import default_transforms, train_default_transforms +if _FIFTYONE_AVAILABLE: + from fiftyone.core.collections import SampleCollection + if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt else: @@ -150,6 +154,47 @@ def predict_load_sample(self, sample: Mapping[str, Any]) -> Mapping[str, Any]: } +class SemanticSegmentationFiftyOneDataSource(FiftyOneDataSource): + + def __init__(self, label_field: str = "ground_truth"): + if not _IMAGE_AVAILABLE: + raise ModuleNotFoundError("Please, pip install -e '.[image]'") + super().__init__(label_field) + self._fiftyone_data = None + + def load_data(self, + data: SampleCollection, + dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + """Takes ``data``, a FiftyOne SampleCollection (generally a + ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and + parses sample filenames and labels from the given label field into a + list of inputs and targets. + """ + self._fiftyone_data = data + filepaths = data.values("filepath") + return [{DefaultDataKeys.INPUT: f} for f in filepaths] + + def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: + img_path = sample[DefaultDataKeys.INPUT] + fo_sample = self._fiftyone_data[img_path] + + img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW + img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field+".segmentation.mask"]) # HxW + + return { + DefaultDataKeys.INPUT: img.float(), + DefaultDataKeys.TARGET: img_labels.float(), + DefaultDataKeys.METADATA: img.shape, + } + + def predict_load_sample(self, sample: Mapping[str, Any]) -> Mapping[str, Any]: + img = torchvision.io.read_image(sample[DefaultDataKeys.INPUT]).float() + return { + DefaultDataKeys.INPUT: img, + DefaultDataKeys.METADATA: img.shape, + } + + class SemanticSegmentationPreprocess(Preprocess): def __init__( @@ -161,6 +206,7 @@ def __init__( image_size: Tuple[int, int] = (196, 196), num_classes: int = None, labels_map: Dict[int, Tuple[int, int, int]] = None, + **data_source_kwargs, ) -> None: """Preprocess pipeline for semantic segmentation tasks. @@ -170,6 +216,8 @@ def __init__( test_transform: Dictionary with the set of transforms to apply during testing. predict_transform: Dictionary with the set of transforms to apply during prediction. image_size: A tuple with the expected output image size. + **data_source_kwargs: Additional arguments passed on to the data + source constructors """ if not _IMAGE_AVAILABLE: raise ModuleNotFoundError("Please, pip install -e '.[image]'") @@ -184,6 +232,7 @@ def __init__( test_transform=test_transform, predict_transform=predict_transform, data_sources={ + DefaultDataSources.FIFTYONE: SemanticSegmentationFiftyOneDataSource(**data_source_kwargs), DefaultDataSources.FILES: SemanticSegmentationPathsDataSource(), DefaultDataSources.FOLDERS: SemanticSegmentationPathsDataSource(), DefaultDataSources.TENSORS: SemanticSegmentationTensorDataSource(), diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 619541fed8..58d2500a2d 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -19,7 +19,10 @@ import flash from flash.core.data.data_source import DefaultDataKeys, ImageLabelsMap from flash.core.data.process import Serializer -from flash.core.utilities.imports import _KORNIA_AVAILABLE, _MATPLOTLIB_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _KORNIA_AVAILABLE, _MATPLOTLIB_AVAILABLE + +if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Segmentation if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -80,3 +83,13 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> torch.Tensor: plt.imshow(labels_vis) plt.show() return labels + + +class FiftyOneSegmentationLabels(SegmentationLabels): + + def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: + preds = sample[DefaultDataKeys.PREDS] + assert len(preds.shape) == 3, preds.shape + labels = torch.argmax(preds, dim=-3).numpy() # HxW + + return Segmentation(mask=labels) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 4c5350af7d..69185a69ad 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -137,7 +137,9 @@ def __init__( decoder: str = "pyav", label_field: str = "ground_truth", ): - super().__init__( + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + VideoClassificationPathsDataSource.__init__( clip_sampler=clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, From 3289ffb59670045ed7bc42c5c10326e4e4ffe338 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 13:16:05 -0400 Subject: [PATCH 07/54] values list, load segmentation dataset in load sample --- flash/core/data/data_source.py | 3 +-- flash/image/detection/data.py | 5 ++--- flash/image/segmentation/data.py | 10 ++++++---- flash/video/classification/data.py | 3 +-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 8276f682ff..d7fc079285 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -487,9 +487,8 @@ def load_data(self, parses sample filenames and labels from the given label field into a list of inputs and targets. """ - filepaths = data.values("filepath") _, label_path = data._get_label_field_path(self.label_field, "label") - targets = data.values(label_path) + filepaths, targets = data.values(["filepath", label_path]) classes = data.default_classes if not classes: diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index eb64ecc7d7..f5ca2b4114 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -127,12 +127,11 @@ def load_data(self, list of inputs and targets. """ data.compute_metadata() - filepaths = data.values("filepath") + + filepaths, widths, heights = data.values(["filepath", "metadata.width", "metadata.height"]) labels = data.values(self.label_field + ".detections.label") bboxes = data.values(self.label_field + ".detections.bounding_box") iscrowds = data.values(self.label_field + ".detections." + self.iscrowd) - widths = data.values("metadata.width") - heights = data.values("metadata.height") classes = data.default_classes if not classes: diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 0eb213777f..18b42e2571 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -42,6 +42,7 @@ from flash.image.segmentation.transforms import default_transforms, train_default_transforms if _FIFTYONE_AVAILABLE: + import fiftyone as fo from fiftyone.core.collections import SampleCollection if _MATPLOTLIB_AVAILABLE: @@ -160,7 +161,7 @@ def __init__(self, label_field: str = "ground_truth"): if not _IMAGE_AVAILABLE: raise ModuleNotFoundError("Please, pip install -e '.[image]'") super().__init__(label_field) - self._fiftyone_data = None + self._fiftyone_dataset_name = None def load_data(self, data: SampleCollection, @@ -170,16 +171,17 @@ def load_data(self, parses sample filenames and labels from the given label field into a list of inputs and targets. """ - self._fiftyone_data = data + self._fiftyone_dataset_name = data.name filepaths = data.values("filepath") return [{DefaultDataKeys.INPUT: f} for f in filepaths] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: img_path = sample[DefaultDataKeys.INPUT] - fo_sample = self._fiftyone_data[img_path] + fo_dataset = fo.load_dataset(self._fiftyone_dataset_name) + fo_sample = fo_dataset[img_path] img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW - img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field+".segmentation.mask"]) # HxW + img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field]["mask"]) # HxW return { DefaultDataKeys.INPUT: img.float(), diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 69185a69ad..c055f90d43 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -151,8 +151,7 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'E label_to_class_mapping = dict(enumerate(data.default_classes)) class_to_label_mapping = {c:l for l,c in label_to_class_mapping.items()} - filepaths = data.values("filepath") - labels = data.values(self.label_field + ".label") + filepaths, labels = data.values(["filepath", self.label_field+".label"]) targets = [class_to_label_mapping[l] for l in labels] labeled_video_paths = list(zip(filepaths, targets)) From 3748dbf6e61ccfa4bc5be52ddb39d841c6e0dd21 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 15:22:30 -0400 Subject: [PATCH 08/54] FiftyOneLabels test --- flash/core/classification.py | 23 ++++++++++++++++++----- tests/core/test_classification.py | 9 ++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 55cfec53bb..27320f9eaf 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -241,10 +241,23 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: ) else: rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) - fo_labels = Classification( - label = str(classes), - confidence = max(probabilities), - logits = logits - ) + if self.multi_label: + classifications = [] + for cls in classes: + fo_cls = Classification( + label = str(cls), + confidence = probabilities[cls], + ) + classifications.append(fo_cls) + fo_labels = Classifications( + classifications = classifications, + logits = logits, + ) + else: + fo_labels = Classification( + label = str(classes), + confidence = max(probabilities), + logits = logits + ) return fo_labels diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index fd4de14d7e..cc2af665a1 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -13,7 +13,7 @@ # limitations under the License. import torch -from flash.core.classification import Classes, Labels, Logits, Probabilities +from flash.core.classification import Classes, FiftyOneLabels, Labels, Logits, Probabilities def test_classification_serializers(): @@ -21,9 +21,13 @@ def test_classification_serializers(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits().serialize(example_output)), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True).serialize(example_output).logits), example_output) assert torch.allclose(torch.tensor(Probabilities().serialize(example_output)), torch.softmax(example_output, -1)) + assert torch.allclose(torch.tensor(FiftyOneLabels().serialize(example_output).confidence), torch.softmax(example_output, -1)[-1]) assert Classes().serialize(example_output) == 2 assert Labels(labels).serialize(example_output) == 'class_3' + assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' + assert FiftyOneLabels().serialize(example_output).label == '2' def test_classification_serializers_multi_label(): @@ -31,9 +35,12 @@ def test_classification_serializers_multi_label(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits(multi_label=True).serialize(example_output)), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True,multi_label=True).serialize(example_output).logits), example_output) assert torch.allclose( torch.tensor(Probabilities(multi_label=True).serialize(example_output)), torch.sigmoid(example_output), ) assert Classes(multi_label=True).serialize(example_output) == [1, 2] + assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] assert Labels(labels, multi_label=True).serialize(example_output) == ['class_2', 'class_3'] + assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize(example_output).classifications] == ['class_2', 'class_3'] From 7ca3d92c1800447703182daafd3035c95688c635 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 15:50:32 -0400 Subject: [PATCH 09/54] serializer and detection tests --- tests/core/test_classification.py | 16 ++++ tests/image/detection/test_data.py | 91 ++++++++++++++++++- .../detection/test_data_model_integration.py | 27 +++++- 3 files changed, 132 insertions(+), 2 deletions(-) diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index cc2af665a1..0893a15204 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -14,6 +14,7 @@ import torch from flash.core.classification import Classes, FiftyOneLabels, Labels, Logits, Probabilities +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE def test_classification_serializers(): @@ -44,3 +45,18 @@ def test_classification_serializers_multi_label(): assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] assert Labels(labels, multi_label=True).serialize(example_output) == ['class_2', 'class_3'] assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize(example_output).classifications] == ['class_2', 'class_3'] + + +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") +def test_classification_serializers_fiftyone(): + example_output = torch.tensor([-0.1, 0.2, 0.3]) # 3 classes + labels = ['class_1', 'class_2', 'class_3'] + + assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True).serialize(example_output).logits), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels().serialize(example_output).confidence), torch.softmax(example_output, -1)[-1]) + assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' + assert FiftyOneLabels().serialize(example_output).label == '2' + assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True,multi_label=True).serialize(example_output).logits), example_output) + assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] + assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize(example_output).classifications] == ['class_2', 'class_3'] + diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index 7e5fc5ad9a..000dcda01d 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -5,12 +5,15 @@ import pytest from flash.core.data.data_source import DefaultDataKeys -from flash.core.utilities.imports import _COCO_AVAILABLE, _IMAGE_AVAILABLE +from flash.core.utilities.imports import _COCO_AVAILABLE, _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE from flash.image.detection.data import ObjectDetectionData if _IMAGE_AVAILABLE: from PIL import Image +if _FIFTYONE_AVAILABLE: + import fiftyone as fo + def _create_dummy_coco_json(dummy_json_path): @@ -77,6 +80,49 @@ def _create_synth_coco_dataset(tmpdir): return train_folder, coco_ann_path +def _create_synth_fiftyone_dataset(tmpdir): + img_dir = Path(tmpdir / "fo_imgs") + img_dir.mkdir() + + Image.new('RGB', (1920, 1080)).save(img_dir / "sample_one.png") + Image.new('RGB', (1920, 1080)).save(img_dir / "sample_two.png") + + dataset = fo.Dataset.from_dir( + img_dir, dataset_type=fo.types.ImageDirectory, + ) + + sample1 = dataset[str(img_dir / "sample_one.png")] + sample2 = dataset[str(img_dir / "sample_two.png")] + + d1 = fo.Detection( + label = "person", + bounding_box = [0.3, 0.4, 0.2, 0.2], + ) + d2 = fo.Detection( + label = "person", + bounding_box = [0.05, 0.10, 0.28, 0.15], + ) + d3 = fo.Detection( + label = "person", + bounding_box = [0.23, 0.14, 0.09, 0.18], + ) + d1["iscrowd"] = 1 + d2["iscrowd"] = 0 + d3["iscrowd"] = 0 + + sample1["ground_truth"] = fo.Detections( + detections=[d1] + ) + sample2["ground_truth"] = fo.Detections( + detections=[d2,d3] + ) + + sample1.save() + sample2.save() + + return dataset + + @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="pycocotools is not installed for testing") def test_image_detector_data_from_coco(tmpdir): @@ -121,3 +167,46 @@ def test_image_detector_data_from_coco(tmpdir): assert imgs[0].shape == (3, 1080, 1920) assert len(labels) == 1 assert list(labels[0].keys()) == ['boxes', 'labels', 'image_id', 'area', 'iscrowd'] + + +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") +def test_image_detector_data_from_fiftyone(tmpdir): + + train_dataset = _create_synth_fiftyone_dataset(tmpdir) + + datamodule = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) + + data = next(iter(datamodule.train_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + + assert len(imgs) == 1 + assert imgs[0].shape == (3, 1080, 1920) + assert len(labels) == 1 + assert list(labels[0].keys()) == ['boxes', 'labels', 'image_id', 'area', 'iscrowd'] + + assert datamodule.val_dataloader() is None + assert datamodule.test_dataloader() is None + + datamodule = ObjectDetectionData.from_fiftyone( + train_dataset=train_dataset, + val_dataset=train_dataset, + test_dataset=train_dataset, + batch_size=1, + num_workers=0, + ) + + data = next(iter(datamodule.val_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + + assert len(imgs) == 1 + assert imgs[0].shape == (3, 1080, 1920) + assert len(labels) == 1 + assert list(labels[0].keys()) == ['boxes', 'labels', 'image_id', 'area', 'iscrowd'] + + data = next(iter(datamodule.test_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + + assert len(imgs) == 1 + assert imgs[0].shape == (3, 1080, 1920) + assert len(labels) == 1 + assert list(labels[0].keys()) == ['boxes', 'labels', 'image_id', 'area', 'iscrowd'] diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index 428a053b75..422b5d4411 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -16,7 +16,7 @@ import pytest import flash -from flash.core.utilities.imports import _COCO_AVAILABLE, _IMAGE_AVAILABLE +from flash.core.utilities.imports import _COCO_AVAILABLE, _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE from flash.image import ObjectDetector from flash.image.detection import ObjectDetectionData @@ -28,6 +28,9 @@ if _COCO_AVAILABLE: from tests.image.detection.test_data import _create_synth_coco_dataset +if _FIFTYONE_AVAILABLE: + from tests.image.detection.test_data import _create_synth_fiftyone_dataset + @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="pycocotools is not installed for testing") @pytest.mark.skipif(not _COCO_AVAILABLE, reason="pycocotools is not installed for testing") @@ -51,3 +54,25 @@ def test_detection(tmpdir, model, backbone): test_images = [str(test_image_one), str(test_image_two)] model.predict(test_images) + +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") +@pytest.mark.parametrize(["model", "backbone"], [("fasterrcnn", "resnet18")]) +def test_detection_fiftyone(tmpdir, model, backbone): + + train_dataset = _create_synth_fiftyone_dataset(tmpdir) + + data = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) + model = ObjectDetector(model=model, backbone=backbone, num_classes=data.num_classes) + + trainer = flash.Trainer(fast_dev_run=True) + + trainer.finetune(model, data) + + test_image_one = os.fspath(tmpdir / "test_one.png") + test_image_two = os.fspath(tmpdir / "test_two.png") + + Image.new('RGB', (512, 512)).save(test_image_one) + Image.new('RGB', (512, 512)).save(test_image_two) + + test_images = [str(test_image_one), str(test_image_two)] + model.predict(test_images) From 95690ac93f4b4bb9671c518941c3428f65651192 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 16:02:53 -0400 Subject: [PATCH 10/54] fiftyone classification tests --- tests/image/classification/test_data.py | 44 ++++++++++++++++++- .../test_data_model_integration.py | 40 ++++++++++++++++- tests/image/detection/test_data.py | 1 + .../detection/test_data_model_integration.py | 1 + 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index ae7159a34f..adb2f493a8 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -21,7 +21,7 @@ from flash.core.data.data_source import DefaultDataKeys from flash.core.data.transforms import ApplyToKeys -from flash.core.utilities.imports import _IMAGE_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE from flash.image import ImageClassificationData if _IMAGE_AVAILABLE: @@ -29,6 +29,9 @@ import torchvision from PIL import Image +if _FIFTYONE_AVAILABLE: + import fiftyone as fo + def _dummy_image_loader(_): return torch.rand(3, 196, 196) @@ -381,3 +384,42 @@ def test_from_data(data, from_function): assert imgs.shape == (2, 3, 196, 196) assert labels.shape == (2, ) assert list(labels.numpy()) == [2, 5] + + +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") +def test_from_fiftyone(tmpdir): + tmpdir = Path(tmpdir) + + (tmpdir / "a").mkdir() + (tmpdir / "b").mkdir() + _rand_image().save(tmpdir / "a_1.png") + _rand_image().save(tmpdir / "b_1.png") + + train_images = [ + str(tmpdir / "a_1.png"), + str(tmpdir / "b_1.png"), + ] + + train_dataset = fo.Dataset.from_dir(str(tmpdir), dataset_type=fo.types.ImageDirectory) + s1 = train_dataset[train_images[0]] + s2 = train_dataset[train_images[1]] + s1["test"] = fo.Classification(label="1") + s2["test"] = fo.Classification(label="2") + s1.save() + s2.save() + + img_data = ImageClassificationData.from_fiftyone( + train_dataset=train_dataset, + label_field="test", + batch_size=2, + num_workers=0, + ) + assert img_data.train_dataloader() is not None + assert img_data.val_dataloader() is None + assert img_data.test_dataloader() is None + + data = next(iter(img_data.train_dataloader())) + imgs, labels = data['input'], data['target'] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, ) + assert sorted(list(labels.numpy())) == [0, 1] diff --git a/tests/image/classification/test_data_model_integration.py b/tests/image/classification/test_data_model_integration.py index 002f445f8e..9414c35f8e 100644 --- a/tests/image/classification/test_data_model_integration.py +++ b/tests/image/classification/test_data_model_integration.py @@ -18,12 +18,15 @@ import torch from flash import Trainer -from flash.core.utilities.imports import _IMAGE_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE from flash.image import ImageClassificationData, ImageClassifier if _IMAGE_AVAILABLE: from PIL import Image +if _FIFTYONE_AVAILABLE: + import fiftyone as fo + def _dummy_image_loader(_): return torch.rand(3, 224, 224) @@ -56,3 +59,38 @@ def test_classification(tmpdir): model = ImageClassifier(num_classes=2, backbone="resnet18") trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True) trainer.finetune(model, datamodule=data, strategy="freeze") + +@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed.") +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") +def test_classification_fiftyone(tmpdir): + tmpdir = Path(tmpdir) + + (tmpdir / "a").mkdir() + (tmpdir / "b").mkdir() + _rand_image().save(tmpdir / "a_1.png") + _rand_image().save(tmpdir / "b_1.png") + + train_images = [ + str(tmpdir / "a_1.png"), + str(tmpdir / "b_1.png"), + ] + + train_dataset = fo.Dataset.from_dir(str(tmpdir), dataset_type=fo.types.ImageDirectory) + s1 = train_dataset[train_images[0]] + s2 = train_dataset[train_images[1]] + s1["test"] = fo.Classification(label="1") + s2["test"] = fo.Classification(label="2") + s1.save() + s2.save() + + data = ImageClassificationData.from_fiftyone( + train_dataset=train_dataset, + label_field="test", + batch_size=2, + num_workers=0, + image_size=(64, 64), + ) + + model = ImageClassifier(num_classes=2, backbone="resnet18") + trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True) + trainer.finetune(model, datamodule=data, strategy="freeze") diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index 000dcda01d..1f132f1c3f 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -169,6 +169,7 @@ def test_image_detector_data_from_coco(tmpdir): assert list(labels[0].keys()) == ['boxes', 'labels', 'image_id', 'area', 'iscrowd'] +@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") def test_image_detector_data_from_fiftyone(tmpdir): diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index 422b5d4411..89be010a95 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -55,6 +55,7 @@ def test_detection(tmpdir, model, backbone): test_images = [str(test_image_one), str(test_image_two)] model.predict(test_images) +@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed for testing") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") @pytest.mark.parametrize(["model", "backbone"], [("fasterrcnn", "resnet18")]) def test_detection_fiftyone(tmpdir, model, backbone): From 5883aae3f43d80964cc0dd2ef6378846a87cfb36 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 17:18:37 -0400 Subject: [PATCH 11/54] segmentation and video tests --- flash/video/classification/data.py | 1 + tests/image/segmentation/test_data.py | 71 +++++++++++- .../image/segmentation/test_serialization.py | 15 ++- tests/video/test_video_classifier.py | 107 +++++++++++++++++- 4 files changed, 188 insertions(+), 6 deletions(-) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index c055f90d43..064e870b41 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -140,6 +140,7 @@ def __init__( if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") VideoClassificationPathsDataSource.__init__( + self, clip_sampler=clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index a45f0a947a..edae4d42a4 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -9,12 +9,15 @@ from flash import Trainer from flash.core.data.data_source import DefaultDataKeys -from flash.core.utilities.imports import _IMAGE_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _IMAGE_AVAILABLE from flash.image import SemanticSegmentation, SemanticSegmentationData, SemanticSegmentationPreprocess if _IMAGE_AVAILABLE: from PIL import Image +if _FIFTYONE_AVAILABLE: + import fiftyone as fo + def build_checkboard(n, m, k=8): x = np.zeros((n, m)) @@ -248,6 +251,72 @@ def test_from_files_warning(self, tmpdir): num_classes=num_classes ) + @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") + def test_from_fiftyone(self, tmpdir): + tmp_dir = Path(tmpdir) + + # create random dummy data + + os.makedirs(str(tmp_dir / "images")) + os.makedirs(str(tmp_dir / "targets")) + + images = [ + str(tmp_dir / "images" / "img1.png"), + str(tmp_dir / "images" / "img2.png"), + str(tmp_dir / "images" / "img3.png"), + ] + + targets = [ + str(tmp_dir / "targets" / "img1.png"), + str(tmp_dir / "targets" / "img2.png"), + str(tmp_dir / "targets" / "img3.png"), + ] + + num_classes: int = 2 + img_size: Tuple[int, int] = (196, 196) + create_random_data(images, targets, img_size, num_classes) + + dataset = fo.Dataset.from_dir( + str(tmp_dir), + dataset_type=fo.types.ImageSegmentationDirectory, + data_path="images", + labels_path="targets", + force_grayscale=True, + ) + + # instantiate the data module + + dm = SemanticSegmentationData.from_fiftyone( + train_dataset=dataset, + val_dataset=dataset, + test_dataset=dataset, + batch_size=2, + num_workers=0, + num_classes=num_classes, + ) + assert dm is not None + assert dm.train_dataloader() is not None + assert dm.val_dataloader() is not None + assert dm.test_dataloader() is not None + + # check training data + data = next(iter(dm.train_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, 196, 196) + + # check val data + data = next(iter(dm.val_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, 196, 196) + + # check test data + data = next(iter(dm.test_dataloader())) + imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, 196, 196) + def test_map_labels(self, tmpdir): tmp_dir = Path(tmpdir) diff --git a/tests/image/segmentation/test_serialization.py b/tests/image/segmentation/test_serialization.py index 3438cde94f..99823ef92d 100644 --- a/tests/image/segmentation/test_serialization.py +++ b/tests/image/segmentation/test_serialization.py @@ -2,7 +2,8 @@ import torch from flash.core.data.data_source import DefaultDataKeys -from flash.image.segmentation.serialization import SegmentationLabels +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE +from flash.image.segmentation.serialization import FiftyOneSegmentationLabels, SegmentationLabels class TestSemanticSegmentationLabels: @@ -35,6 +36,18 @@ def test_serialize(self): assert classes[1, 2] == 1 assert classes[0, 1] == 3 + @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") + def test_serialize_fiftyone(self): + serial = FiftyOneSegmentationLabels() + + sample = torch.zeros(5, 2, 3) + sample[1, 1, 2] = 1 # add peak in class 2 + sample[3, 0, 1] = 1 # add peak in class 4 + + segmentation = serial.serialize({DefaultDataKeys.PREDS: sample}) + assert segmentation.mask[1, 2] == 1 + assert segmentation.mask[0, 1] == 3 + # TODO: implement me def test_create_random_labels(self): pass diff --git a/tests/video/test_video_classifier.py b/tests/video/test_video_classifier.py index caf445af65..f188e3f968 100644 --- a/tests/video/test_video_classifier.py +++ b/tests/video/test_video_classifier.py @@ -13,6 +13,7 @@ # limitations under the License. import contextlib import os +from pathlib import Path import tempfile import pytest @@ -20,7 +21,10 @@ from torch.utils.data import SequentialSampler import flash -from flash.core.utilities.imports import _VIDEO_AVAILABLE +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _VIDEO_AVAILABLE + +if _FIFTYONE_AVAILABLE: + import fiftyone as fo if _VIDEO_AVAILABLE: import kornia.augmentation as K @@ -45,7 +49,7 @@ def create_dummy_video_frames(num_frames: int, height: int, width: int): # https://github.com/facebookresearch/pytorchvideo/blob/4feccb607d7a16933d485495f91d067f177dd8db/tests/utils.py#L33 @contextlib.contextmanager -def temp_encoded_video(num_frames: int, fps: int, height=10, width=10, prefix=None): +def temp_encoded_video(num_frames: int, fps: int, height=10, width=10, prefix=None, directory=None): """ Creates a temporary lossless, mp4 video with synthetic content. Uses a context which deletes the video after exit. @@ -54,7 +58,7 @@ def temp_encoded_video(num_frames: int, fps: int, height=10, width=10, prefix=No video_codec = "libx264rgb" options = {"crf": "0"} data = create_dummy_video_frames(num_frames, height, width) - with tempfile.NamedTemporaryFile(prefix=prefix, suffix=".mp4") as f: + with tempfile.NamedTemporaryFile(prefix=prefix, suffix=".mp4", dir=directory) as f: f.close() io.write_video(f.name, data, fps=fps, video_codec=video_codec, options=options) yield f.name, thwc_to_cthw(data).to(torch.float32) @@ -94,8 +98,33 @@ def mock_encoded_video_dataset_file(): yield f.name, label_videos, video_duration +@contextlib.contextmanager +def mock_encoded_video_dataset_folder(tmpdir): + """ + Creates a temporary mock encoded video directory tree with 2 videos labeled 1, 2. + Returns a directory that to this mock encoded video dataset and the video duration in seconds. + """ + num_frames = 10 + fps = 5 + + tmp_dir = Path(tmpdir) + os.makedirs(str(tmp_dir / "c1")) + os.makedirs(str(tmp_dir / "c2")) + + with temp_encoded_video(num_frames=num_frames, fps=fps, directory=str(tmp_dir / "c1")) as ( + video_file_name_1, + data_1, + ): + with temp_encoded_video(num_frames=num_frames, fps=fps, directory=str(tmp_dir / "c2")) as ( + video_file_name_2, + data_2, + ): + video_duration = num_frames / fps + yield str(tmp_dir), video_duration + + @pytest.mark.skipif(not _VIDEO_AVAILABLE, reason="PyTorchVideo isn't installed.") -def test_image_classifier_finetune(tmpdir): +def test_video_classifier_finetune(tmpdir): with mock_encoded_video_dataset_file() as ( mock_csv, @@ -158,3 +187,73 @@ def test_image_classifier_finetune(tmpdir): trainer = flash.Trainer(fast_dev_run=True) trainer.finetune(model, datamodule=datamodule) + + +@pytest.mark.skipif(not _VIDEO_AVAILABLE, reason="PyTorchVideo isn't installed.") +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") +def test_video_classifier_finetune_fiftyone(tmpdir): + + with mock_encoded_video_dataset_folder(tmpdir) as ( + dir_name, + total_duration, + ): + + half_duration = total_duration / 2 - 1e-9 + + train_dataset = fo.Dataset.from_dir( + dir_name, + dataset_type=fo.types.VideoClassificationDirectoryTree, + ) + datamodule = VideoClassificationData.from_fiftyone( + train_dataset=train_dataset, + clip_sampler="uniform", + clip_duration=half_duration, + video_sampler=SequentialSampler, + decode_audio=False, + ) + + for sample in datamodule.train_dataset.data: + expected_t_shape = 5 + assert sample["video"].shape[1] == expected_t_shape + + assert len(VideoClassifier.available_backbones()) > 5 + + train_transform = { + "post_tensor_transform": Compose([ + ApplyTransformToKey( + key="video", + transform=Compose([ + UniformTemporalSubsample(8), + RandomShortSideScale(min_size=256, max_size=320), + RandomCrop(244), + RandomHorizontalFlip(p=0.5), + ]), + ), + ]), + "per_batch_transform_on_device": Compose([ + ApplyTransformToKey( + key="video", + transform=K.VideoSequential( + K.Normalize(torch.tensor([0.45, 0.45, 0.45]), torch.tensor([0.225, 0.225, 0.225])), + K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.0), + data_format="BCTHW", + same_on_frame=False + ) + ), + ]), + } + + datamodule = VideoClassificationData.from_fiftyone( + train_dataset=train_dataset, + clip_sampler="uniform", + clip_duration=half_duration, + video_sampler=SequentialSampler, + decode_audio=False, + train_transform=train_transform + ) + + model = VideoClassifier(num_classes=datamodule.num_classes, pretrained=False) + + trainer = flash.Trainer(fast_dev_run=True) + + trainer.finetune(model, datamodule=datamodule) From ab34980dcd16ed936c3885b2c54e0aad2268c249 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 18:12:44 -0400 Subject: [PATCH 12/54] add detections serializiation test --- tests/image/detection/test_serialization.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/image/detection/test_serialization.py diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py new file mode 100644 index 0000000000..9da284c628 --- /dev/null +++ b/tests/image/detection/test_serialization.py @@ -0,0 +1,26 @@ +import pytest +import torch + +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE +from flash.image.detection.serialization import FiftyOneDetectionLabels + + +@pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") +class TestFiftyOneDetectionLabels: + + def test_smoke(self): + serial = FiftyOneDetectionLabels() + assert serial is not None + + def test_serialize_fiftyone(self): + serial = FiftyOneDetectionLabels() + + sample = [{ + "boxes": [torch.tensor(20), torch.tensor(30), torch.tensor(40), torch.tensor(50)], + "labels": torch.tensor(0), + "scores": torch.tensor(0.5), + }] + + detections = serial.serialize(sample) + assert len(detections.detections) == 1 + assert detections.detections[0].bounding_box == [20,30,20,20] From 036a28b07db4a838ebdb366497517bebc6d6f680 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 18:23:43 -0400 Subject: [PATCH 13/54] cleanup --- flash/core/classification.py | 45 +++++++------ flash/core/data/data_module.py | 8 +-- flash/core/data/data_source.py | 20 +++--- flash/image/classification/data.py | 4 -- flash/image/detection/data.py | 76 +++++++++++----------- flash/image/detection/serialization.py | 27 ++++---- flash/image/segmentation/data.py | 16 ++--- flash/image/segmentation/serialization.py | 3 +- flash/video/classification/data.py | 79 +++++++++++++---------- 9 files changed, 141 insertions(+), 137 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 27320f9eaf..4c1bd8fb81 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -165,16 +165,14 @@ def serialize(self, sample: Any) -> Union[int, List[int], str, List[str]]: class FiftyOneLabels(ClassificationSerializer): - """A :class:`.Serializer` which converts the model outputs (assumed to be logits) to the label of the - argmax classification all stored as a FiftyOne Classification label. + """A :class:`.Serializer` which converts the model outputs (assumed to be logits) to FiftyOne classification format. + Args: labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not provided, will attempt to get them from the :class:`.LabelsState`. - multi_label: If true, treats outputs as multi label logits and creates - FiftyOne Classifications. + multi_label: If true, treats outputs as multi label logits. threshold: The threshold to use for multi_label classification. - store_logits: Boolean determining whether to store logits in - the FiftyOne labels + store_logits: Boolean determining whether to store logits in the FiftyOne labels """ def __init__( @@ -223,41 +221,42 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: if labels is not None: if self.multi_label: classifications = [] - for cls in classes: + for idx in classes: fo_cls = Classification( - label = labels[cls], - confidence = probabilities[cls], + label=labels[idx], + confidence=probabilities[idx], ) classifications.append(fo_cls) fo_labels = Classifications( - classifications = classifications, - logits = logits, + classifications=classifications, + logits=logits, ) else: fo_labels = Classification( - label = labels[classes], - confidence = max(probabilities), - logits = logits, + label=labels[classes], + confidence=max(probabilities), + logits=logits, ) else: - rank_zero_warn("No LabelsState was found, this serializer will act as a Classes serializer.", UserWarning) + rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) + if self.multi_label: classifications = [] - for cls in classes: + for idx in classes: fo_cls = Classification( - label = str(cls), - confidence = probabilities[cls], + label=str(idx), + confidence=probabilities[idx], ) classifications.append(fo_cls) fo_labels = Classifications( - classifications = classifications, - logits = logits, + classifications=classifications, + logits=logits, ) else: fo_labels = Classification( - label = str(classes), - confidence = max(probabilities), - logits = logits + label=str(classes), + confidence=max(probabilities), + logits=logits, ) return fo_labels diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 34a46b8d72..3b1937a2c7 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -1056,10 +1056,10 @@ def from_fiftyone( from the passed or constructed :class:`~flash.core.data.process.Preprocess`. Args: - train_dataset: The FiftyOne Dataset containing the train data. - val_dataset: The FiftyOne Dataset containing the validation data. - test_dataset: The FiftyOne Dataset containing the test data. - predict_dataset: The FiftyOne Dataset containing the predict data. + train_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the train data. + val_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the validation data. + test_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the test data. + predict_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the predict data. train_transform: The dictionary of transforms to use during training which maps :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. val_transform: The dictionary of transforms to use during validation which maps diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index d7fc079285..1a593e2685 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -471,7 +471,7 @@ class NumpyDataSource(SequenceDataSource[np.ndarray]): class FiftyOneDataSource(DataSource[SampleCollection]): """The ``FiftyOneDataSource`` expects the input to - :meth:`~flash.core.data.data_source.DataSource.load_data` to be FiftyOne Dataset objects.""" + :meth:`~flash.core.data.data_source.DataSource.load_data` to be a ``fiftyone.core.collections.SampleCollection``.""" def __init__(self, label_field: str = "ground_truth"): if not _FIFTYONE_AVAILABLE: @@ -482,16 +482,13 @@ def __init__(self, label_field: str = "ground_truth"): def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - """Takes ``data``, a FiftyOne SampleCollection (generally a - ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and - parses sample filenames and labels from the given label field into a - list of inputs and targets. - """ _, label_path = data._get_label_field_path(self.label_field, "label") filepaths, targets = data.values(["filepath", label_path]) classes = data.default_classes - if not classes: + if classes is None: + classes = data.classes.get(self.label_field, None) + if classes is None: classes = data.distinct(label_path) if dataset is not None: @@ -502,11 +499,12 @@ def load_data(self, if targets and isinstance(targets[0], list): to_idx = lambda t: [class_to_idx[x] for x in t] - - data = zip(filepaths, targets) - return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t)} for f,t in data] + return [ + {DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t)} + for f, t in zip(filepaths, targets) + ] def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] diff --git a/flash/image/classification/data.py b/flash/image/classification/data.py index ec90e1e8dc..31088dc210 100644 --- a/flash/image/classification/data.py +++ b/flash/image/classification/data.py @@ -51,10 +51,6 @@ def __init__( image_size: Tuple[int, int] = (196, 196), **data_source_kwargs, ): - """ - ``data_source_kwargs`` are source-specific keyword arguments that are - passed to the ``DataSource`` constructors - """ self.image_size = image_size super().__init__( diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index f5ca2b4114..15f67c845e 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -95,56 +95,41 @@ def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: class ObjectDetectionFiftyOneDataSource(ImageFiftyOneDataSource): - def __init__(self, label_field: str = "ground_truth", iscrowd: str = "attributes.iscrowd.value"): - """Constructs an ObjectDetectionFiftyOneDataSource from a FiftyOne - SampleCollection using the given fields - - Args: - label_field: label field name containing information to construct - targets - iscrowd: name of the subfield of detections that stores iscrowd - information - """ + def __init__(self, label_field: str = "ground_truth", iscrowd: str = "iscrowd"): super().__init__(label_field=label_field) self.iscrowd = iscrowd - def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): - xmin *= img_w - ymin *= img_h - box_w *= img_w - box_h *= img_h - xmax = xmin + box_w - ymax = ymin + box_h - output_bbox = [xmin, ymin, xmax, ymax] - return output_bbox, box_w*box_h - def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Dict[str, Any]]: - """Takes ``data``, a FiftyOne SampleCollection (generally a - ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and - parses sample filenames and detections from the given label field into a - list of inputs and targets. - """ data.compute_metadata() - filepaths, widths, heights = data.values(["filepath", "metadata.width", "metadata.height"]) - labels = data.values(self.label_field + ".detections.label") - bboxes = data.values(self.label_field + ".detections.bounding_box") - iscrowds = data.values(self.label_field + ".detections." + self.iscrowd) + filepaths, widths, heights, labels, bboxes, iscrowds = data.values( + [ + "filepath", + "metadata.width", + "metadata.height", + self.label_field + ".detections.label", + self.label_field + ".detections.bounding_box", + self.label_field + ".detections." + self.iscrowd, + ] + ) classes = data.default_classes - if not classes: + if classes is None: + classes = data.classes.get(self.label_field, None) + if classes is None: classes = data.distinct(self.label_field + ".detections.label") + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} if dataset is not None: dataset.num_classes = len(classes) - data = zip(filepaths, widths, heights, labels, bboxes, iscrowds) - output_data = [] img_id = 1 - for fp, w, h, sample_labs, sample_boxes, sample_iscrowd in data: + for fp, w, h, sample_labs, sample_boxes, sample_iscrowd in zip( + filepaths, widths, heights, labels, bboxes, iscrowds + ): output_boxes = [] output_labs = [] output_iscrowd = [] @@ -173,6 +158,16 @@ def load_data(self, return output_data + def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): + xmin *= img_w + ymin *= img_h + box_w *= img_w + box_h *= img_h + xmax = xmin + box_w + ymax = ymin + box_h + output_bbox = [xmin, ymin, xmax, ymax] + return output_bbox, box_w * box_h + class ObjectDetectionPreprocess(Preprocess): @@ -184,9 +179,14 @@ def __init__( predict_transform: Optional[Dict[str, Callable]] = None, **data_source_kwargs, ): - """ - ``data_source_kwargs`` are source-specific keyword arguments that are - passed to the ``DataSource`` constructors + """Preprocess pipeline for object detection tasks. + + Args: + train_transform: Dictionary with the set of transforms to apply during training. + val_transform: Dictionary with the set of transforms to apply during validation. + test_transform: Dictionary with the set of transforms to apply during testing. + predict_transform: Dictionary with the set of transforms to apply during prediction. + **data_source_kwargs: Additional arguments passed on to the data source constructors. """ super().__init__( train_transform=train_transform, @@ -194,9 +194,7 @@ def __init__( test_transform=test_transform, predict_transform=predict_transform, data_sources={ - DefaultDataSources.FIFTYONE: ObjectDetectionFiftyOneDataSource( - **data_source_kwargs - ), + DefaultDataSources.FIFTYONE: ObjectDetectionFiftyOneDataSource(**data_source_kwargs), DefaultDataSources.FILES: ImagePathsDataSource(), DefaultDataSources.FOLDERS: ImagePathsDataSource(), "coco": COCODataSource(), diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 9df4535b7d..059265a206 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -21,15 +21,15 @@ if _FIFTYONE_AVAILABLE: from fiftyone.core.labels import Detection, Detections +else: + Detection, Detections = None, None + class FiftyOneDetectionLabels(Serializer): - """A :class:`.Serializer` which converts the model outputs to a FiftyOne Detections label. + """A :class:`.Serializer` which converts the model outputs to FiftyOne detection format. """ - def __init__( - self, - labels: Optional[List[str]] = None, - ): + def __init__(self, labels: Optional[List[str]] = None): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") super().__init__() @@ -40,7 +40,6 @@ def __init__( def serialize(self, sample: List[Dict[str, Any]]) -> Detections: labels = None - if self._labels is not None: labels = self._labels else: @@ -48,13 +47,13 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: if state is not None: labels = state.labels else: - rank_zero_warn("No LabelsState was found, this serializer will return integer class labels.", UserWarning) + rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) detections = [] for det in sample: xmin, ymin, xmax, ymax = [c.tolist() for c in det["boxes"]] - box = [xmin, ymin, xmax-xmin, ymax-ymin] + box = [xmin, ymin, xmax - xmin, ymax - ymin] label = det["labels"].tolist() if labels is not None: @@ -63,11 +62,13 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: label = str(int(label)) score = det["scores"].tolist() - detections.append(Detection( - label = label, - bounding_box = box, - confidence = score, - )) + detections.append( + Detection( + label=label, + bounding_box=box, + confidence=score, + ) + ) return Detections(detections=detections) diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 18b42e2571..db3bdcf394 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -44,6 +44,8 @@ if _FIFTYONE_AVAILABLE: import fiftyone as fo from fiftyone.core.collections import SampleCollection +else: + fo, SampleCollection = None, None if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -166,14 +168,11 @@ def __init__(self, label_field: str = "ground_truth"): def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - """Takes ``data``, a FiftyOne SampleCollection (generally a - ``fiftyone.core.dataset.Dataset`` or ``fiftyone.core.view.View``), and - parses sample filenames and labels from the given label field into a - list of inputs and targets. + """Takes a ``fiftyone.core.collections.SampleCollection`` and parses sample filenames and + labels from the given label field into a list of inputs and targets. """ self._fiftyone_dataset_name = data.name - filepaths = data.values("filepath") - return [{DefaultDataKeys.INPUT: f} for f in filepaths] + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: img_path = sample[DefaultDataKeys.INPUT] @@ -181,7 +180,7 @@ def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Ten fo_sample = fo_dataset[img_path] img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW - img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field]["mask"]) # HxW + img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW return { DefaultDataKeys.INPUT: img.float(), @@ -218,8 +217,7 @@ def __init__( test_transform: Dictionary with the set of transforms to apply during testing. predict_transform: Dictionary with the set of transforms to apply during prediction. image_size: A tuple with the expected output image size. - **data_source_kwargs: Additional arguments passed on to the data - source constructors + **data_source_kwargs: Additional arguments passed on to the data source constructors. """ if not _IMAGE_AVAILABLE: raise ModuleNotFoundError("Please, pip install -e '.[image]'") diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 58d2500a2d..7c09c87177 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -23,6 +23,8 @@ if _FIFTYONE_AVAILABLE: from fiftyone.core.labels import Segmentation +else: + Segmentation = None if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -91,5 +93,4 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: preds = sample[DefaultDataKeys.PREDS] assert len(preds.shape) == 3, preds.shape labels = torch.argmax(preds, dim=-3).numpy() # HxW - return Segmentation(mask=labels) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index c055f90d43..88d6f9305c 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -21,10 +21,10 @@ from flash.core.data.data_module import DataModule from flash.core.data.data_source import ( - DefaultDataKeys, - DefaultDataSources, + DefaultDataKeys, + DefaultDataSources, FiftyOneDataSource, - LabelsState, + LabelsState, PathsDataSource, ) from flash.core.data.process import Preprocess @@ -61,7 +61,7 @@ _PYTORCHVIDEO_DATA = Dict[str, Union[str, torch.Tensor, int, float, List]] -class VideoClassificationPathsDataSource(PathsDataSource): +class _VideoClassificationMixin(object): def __init__( self, @@ -70,26 +70,11 @@ def __init__( decode_audio: bool = True, decoder: str = "pyav", ): - super().__init__(extensions=("mp4", "avi")) self.clip_sampler = clip_sampler self.video_sampler = video_sampler self.decode_audio = decode_audio self.decoder = decoder - def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': - ds: EncodedVideoDataset = labeled_encoded_video_dataset( - pathlib.Path(data), - self.clip_sampler, - video_sampler=self.video_sampler, - decode_audio=self.decode_audio, - decoder=self.decoder, - ) - if self.training: - label_to_class_mapping = {p[1]: p[0].split("/")[-2] for p in ds._labeled_videos._paths_and_labels} - self.set_state(LabelsState(label_to_class_mapping)) - dataset.num_classes = len(np.unique([s[1]['label'] for s in ds._labeled_videos])) - return ds - def _encoded_video_to_dict(self, video) -> Dict[str, Any]: ( clip_start, @@ -123,11 +108,43 @@ def _encoded_video_to_dict(self, video) -> Dict[str, Any]: } if audio_samples is not None else {}), } + +class VideoClassificationPathsDataSource(PathsDataSource, _VideoClassificationMixin): + + def __init__( + self, + clip_sampler: 'ClipSampler', + video_sampler: Type[Sampler] = torch.utils.data.RandomSampler, + decode_audio: bool = True, + decoder: str = "pyav", + ): + super().__init__(extensions=("mp4", "avi")) + _VideoClassificationMixin.__init__( + clip_sampler, + video_sampler=video_sampler, + decode_audio=decode_audio, + decoder=decoder, + ) + + def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': + ds: EncodedVideoDataset = labeled_encoded_video_dataset( + pathlib.Path(data), + self.clip_sampler, + video_sampler=self.video_sampler, + decode_audio=self.decode_audio, + decoder=self.decoder, + ) + if self.training: + label_to_class_mapping = {p[1]: p[0].split("/")[-2] for p in ds._labeled_videos._paths_and_labels} + self.set_state(LabelsState(label_to_class_mapping)) + dataset.num_classes = len(np.unique([s[1]['label'] for s in ds._labeled_videos])) + return ds + def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) -class VideoClassificationFiftyOneDataSource(FiftyOneDataSource, VideoClassificationPathsDataSource): +class VideoClassificationFiftyOneDataSource(FiftyOneDataSource, _VideoClassificationMixin): def __init__( self, @@ -139,8 +156,9 @@ def __init__( ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") - VideoClassificationPathsDataSource.__init__( - clip_sampler=clip_sampler, + + _VideoClassificationMixin.__init__( + clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, decoder=decoder, @@ -149,16 +167,12 @@ def __init__( def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': label_to_class_mapping = dict(enumerate(data.default_classes)) - class_to_label_mapping = {c:l for l,c in label_to_class_mapping.items()} - - filepaths, labels = data.values(["filepath", self.label_field+".label"]) + class_to_label_mapping = {c: l for l, c in label_to_class_mapping.items()} + filepaths, labels = data.values(["filepath", self.label_field + ".label"]) targets = [class_to_label_mapping[l] for l in labels] - labeled_video_paths = list(zip(filepaths, targets)) - labeled_video_paths = LabeledVideoPaths( - labeled_video_paths - ) + labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) ds: EncodedVideoDataset = EncodedVideoDataset( labeled_video_paths, @@ -172,6 +186,9 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'E dataset.num_classes = len(class_to_label_mapping) return ds + def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: + return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) + class VideoClassificationPreprocess(Preprocess): @@ -189,10 +206,6 @@ def __init__( decoder: str = "pyav", **data_source_kwargs, ): - """ - ``data_source_kwargs`` are source-specific keyword arguments that are - passed to the ``DataSource`` constructors - """ self.clip_sampler = clip_sampler self.clip_duration = clip_duration self.clip_sampler_kwargs = clip_sampler_kwargs From d4616b137481756cc5a91a32e76fd289bc165a84 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 18:25:13 -0400 Subject: [PATCH 14/54] fix test --- tests/core/test_classification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index 0893a15204..4c3b0cb8c3 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import pytest + import torch from flash.core.classification import Classes, FiftyOneLabels, Labels, Logits, Probabilities From 51d8046b72ebc675a362804490488a28f9a4d721 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 18:34:57 -0400 Subject: [PATCH 15/54] inherit fiftyonedatasource --- flash/video/classification/data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 70261aa05e..7cba6c86b8 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -157,7 +157,7 @@ def __init__( ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") - + FiftyOneDataSource.__init__(self, label_field) _VideoClassificationMixin.__init__( self, clip_sampler, @@ -165,7 +165,6 @@ def __init__( decode_audio=decode_audio, decoder=decoder, ) - self.label_field = label_field def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': label_to_class_mapping = dict(enumerate(data.default_classes)) From 68e1ce38247fde6dd94023abedf3b82a8d270cc1 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 19:09:44 -0400 Subject: [PATCH 16/54] tweaks --- flash/core/classification.py | 2 ++ flash/image/data.py | 1 + flash/image/detection/data.py | 11 ++--------- flash/image/segmentation/data.py | 17 +++++++++-------- flash/video/classification/data.py | 5 +---- 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 4c1bd8fb81..508f19dc1e 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -25,6 +25,8 @@ if _FIFTYONE_AVAILABLE: from fiftyone.core.labels import Classification, Classifications +else: + Classification, Classifications = None, None def binary_cross_entropy_with_logits(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: diff --git a/flash/image/data.py b/flash/image/data.py index d74689ddee..e82f5af0f6 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -48,6 +48,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> sample[DefaultDataKeys.INPUT] = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) return sample + class ImageFiftyOneDataSource(FiftyOneDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 15f67c845e..1ea1305762 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -27,6 +27,8 @@ if _FIFTYONE_AVAILABLE: from fiftyone.core.collections import SampleCollection +else: + SampleCollection = None if _TORCHVISION_AVAILABLE: from torchvision.datasets.folder import default_loader @@ -179,15 +181,6 @@ def __init__( predict_transform: Optional[Dict[str, Callable]] = None, **data_source_kwargs, ): - """Preprocess pipeline for object detection tasks. - - Args: - train_transform: Dictionary with the set of transforms to apply during training. - val_transform: Dictionary with the set of transforms to apply during validation. - test_transform: Dictionary with the set of transforms to apply during testing. - predict_transform: Dictionary with the set of transforms to apply during prediction. - **data_source_kwargs: Additional arguments passed on to the data source constructors. - """ super().__init__( train_transform=train_transform, val_transform=val_transform, diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index db3bdcf394..208bbb1a4a 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -162,22 +162,23 @@ class SemanticSegmentationFiftyOneDataSource(FiftyOneDataSource): def __init__(self, label_field: str = "ground_truth"): if not _IMAGE_AVAILABLE: raise ModuleNotFoundError("Please, pip install -e '.[image]'") - super().__init__(label_field) - self._fiftyone_dataset_name = None + + super().__init__(label_field=label_field) + self._fo_dataset = None + self._fo_dataset_name = None def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - """Takes a ``fiftyone.core.collections.SampleCollection`` and parses sample filenames and - labels from the given label field into a list of inputs and targets. - """ - self._fiftyone_dataset_name = data.name + self._fo_dataset_name = data.name return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: + if self._fo_dataset is None: + self._fo_dataset = fo.load_dataset(self._fo_dataset_name) + img_path = sample[DefaultDataKeys.INPUT] - fo_dataset = fo.load_dataset(self._fiftyone_dataset_name) - fo_sample = fo_dataset[img_path] + fo_sample = self._fo_dataset[img_path] img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 70261aa05e..b588b73333 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -155,9 +155,7 @@ def __init__( decoder: str = "pyav", label_field: str = "ground_truth", ): - if not _FIFTYONE_AVAILABLE: - raise ModuleNotFoundError("Please, run `pip install fiftyone`.") - + super().__init__(label_field=label_field) _VideoClassificationMixin.__init__( self, clip_sampler, @@ -165,7 +163,6 @@ def __init__( decode_audio=decode_audio, decoder=decoder, ) - self.label_field = label_field def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': label_to_class_mapping = dict(enumerate(data.default_classes)) From 78e033a348a369a86ea4f7cc876dbd0a2a9a187a Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 19:17:53 -0400 Subject: [PATCH 17/54] fix class index --- flash/core/data/data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 1a593e2685..fa55762526 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -486,7 +486,7 @@ def load_data(self, filepaths, targets = data.values(["filepath", label_path]) classes = data.default_classes - if classes is None: + if not classes: classes = data.classes.get(self.label_field, None) if classes is None: classes = data.distinct(label_path) From 073ce6d98aeec2f1213321ddda9b81c2fae8ba8d Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 19:28:07 -0400 Subject: [PATCH 18/54] adding helper functions for common operations --- flash/core/data/data_source.py | 32 +++++++++++++++++++++++++----- flash/image/detection/data.py | 16 ++++++++------- flash/image/segmentation/data.py | 10 ++++++++-- flash/video/classification/data.py | 14 ++++++++++--- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 1a593e2685..d1039bdf9b 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -479,17 +479,19 @@ def __init__(self, label_field: str = "ground_truth"): super().__init__() self.label_field = label_field + @property + def label_cls(self): + return None + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + self._validate(data) + _, label_path = data._get_label_field_path(self.label_field, "label") filepaths, targets = data.values(["filepath", label_path]) - classes = data.default_classes - if classes is None: - classes = data.classes.get(self.label_field, None) - if classes is None: - classes = data.distinct(label_path) + classes = self._get_classes(data) if dataset is not None: dataset.num_classes = len(classes) @@ -508,3 +510,23 @@ def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] + + def _validate(self, data): + if self.label_cls is None: + return + + label_type = data._get_label_field_type(self.label_field) + if not issubclass(label_type, self.label_cls): + raise ValueError("Expected field '%s' to have type %s; found %s" % (self.label_field, self.label_cls, label_type)) + + def _get_classes(self, data): + classes = data.classes.get(self.label_field, None) + + if classes is None: + classes = data.default_classes + + if classes is None: + _, label_path = data._get_label_field_path(self.label_field, "label") + classes = data.distinct(label_path) + + return classes diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 1ea1305762..ceabeb7a86 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -26,9 +26,10 @@ from pycocotools.coco import COCO if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Detections from fiftyone.core.collections import SampleCollection else: - SampleCollection = None + Detections, SampleCollection = None, None if _TORCHVISION_AVAILABLE: from torchvision.datasets.folder import default_loader @@ -101,9 +102,15 @@ def __init__(self, label_field: str = "ground_truth", iscrowd: str = "iscrowd"): super().__init__(label_field=label_field) self.iscrowd = iscrowd + @property + def label_cls(self): + return Detections + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Dict[str, Any]]: + self._validate(data) + data.compute_metadata() filepaths, widths, heights, labels, bboxes, iscrowds = data.values( @@ -117,12 +124,7 @@ def load_data(self, ] ) - classes = data.default_classes - if classes is None: - classes = data.classes.get(self.label_field, None) - if classes is None: - classes = data.distinct(self.label_field + ".detections.label") - + classes = self._get_classes(data) class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} if dataset is not None: dataset.num_classes = len(classes) diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 208bbb1a4a..fc7e76b566 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -42,10 +42,10 @@ from flash.image.segmentation.transforms import default_transforms, train_default_transforms if _FIFTYONE_AVAILABLE: - import fiftyone as fo + from fiftyone.core.labels import Segmentation from fiftyone.core.collections import SampleCollection else: - fo, SampleCollection = None, None + Segmentation, SampleCollection = None, None if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -167,9 +167,15 @@ def __init__(self, label_field: str = "ground_truth"): self._fo_dataset = None self._fo_dataset_name = None + @property + def label_cls(self): + return Segmentation + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + self._validate(data) + self._fo_dataset_name = data.name return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index b588b73333..b62aa6ebfe 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -36,9 +36,10 @@ ) if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Classification from fiftyone.core.collections import SampleCollection else: - SampleCollection = None + Classification, SampleCollection = None, None if _KORNIA_AVAILABLE: import kornia.augmentation as K @@ -164,8 +165,15 @@ def __init__( decoder=decoder, ) + @property + def label_cls(self): + return Classification + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': - label_to_class_mapping = dict(enumerate(data.default_classes)) + self._validate(data) + + classes = self._get_classes(data) + label_to_class_mapping = dict(enumerate(classes)) class_to_label_mapping = {c: l for l, c in label_to_class_mapping.items()} filepaths, labels = data.values(["filepath", self.label_field + ".label"]) @@ -182,7 +190,7 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'E ) if self.training: self.set_state(LabelsState(label_to_class_mapping)) - dataset.num_classes = len(class_to_label_mapping) + dataset.num_classes = len(classes) return ds def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: From e57ca24c7c86e3b6c76d61d42413c46afd1c7b2f Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 19:59:18 -0400 Subject: [PATCH 19/54] updating interface --- flash/core/data/data_source.py | 4 ++-- flash/image/classification/data.py | 2 +- flash/image/detection/data.py | 12 ++++++++---- flash/image/segmentation/data.py | 2 +- flash/video/classification/data.py | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index d1039bdf9b..1a69d1de65 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -522,10 +522,10 @@ def _validate(self, data): def _get_classes(self, data): classes = data.classes.get(self.label_field, None) - if classes is None: + if not classes: classes = data.default_classes - if classes is None: + if not classes: _, label_path = data._get_label_field_path(self.label_field, "label") classes = data.distinct(label_path) diff --git a/flash/image/classification/data.py b/flash/image/classification/data.py index 31088dc210..6fb400cef3 100644 --- a/flash/image/classification/data.py +++ b/flash/image/classification/data.py @@ -49,7 +49,7 @@ def __init__( test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, image_size: Tuple[int, int] = (196, 196), - **data_source_kwargs, + **data_source_kwargs: Any, ): self.image_size = image_size diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index ceabeb7a86..a2b7940a14 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -16,10 +16,10 @@ from flash.core.data.callback import BaseDataFetcher from flash.core.data.data_module import DataModule -from flash.core.data.data_source import DataSource, DefaultDataKeys, DefaultDataSources +from flash.core.data.data_source import DataSource, DefaultDataKeys, DefaultDataSources, FiftyOneDataSource from flash.core.data.process import Preprocess, Serializer from flash.core.utilities.imports import _COCO_AVAILABLE, _FIFTYONE_AVAILABLE, _TORCHVISION_AVAILABLE -from flash.image.data import ImagePathsDataSource, ImageFiftyOneDataSource +from flash.image.data import ImagePathsDataSource from flash.image.detection.transforms import default_transforms if _COCO_AVAILABLE: @@ -96,7 +96,7 @@ def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return sample -class ObjectDetectionFiftyOneDataSource(ImageFiftyOneDataSource): +class ObjectDetectionFiftyOneDataSource(FiftyOneDataSource): def __init__(self, label_field: str = "ground_truth", iscrowd: str = "iscrowd"): super().__init__(label_field=label_field) @@ -162,6 +162,10 @@ def load_data(self, return output_data + def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: + sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + return sample + def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): xmin *= img_w ymin *= img_h @@ -181,7 +185,7 @@ def __init__( val_transform: Optional[Dict[str, Callable]] = None, test_transform: Optional[Dict[str, Callable]] = None, predict_transform: Optional[Dict[str, Callable]] = None, - **data_source_kwargs, + **data_source_kwargs: Any, ): super().__init__( train_transform=train_transform, diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index fc7e76b566..fd746dc837 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -214,7 +214,7 @@ def __init__( image_size: Tuple[int, int] = (196, 196), num_classes: int = None, labels_map: Dict[int, Tuple[int, int, int]] = None, - **data_source_kwargs, + **data_source_kwargs: Any, ) -> None: """Preprocess pipeline for semantic segmentation tasks. diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index b62aa6ebfe..ed80e8fa17 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -211,7 +211,7 @@ def __init__( video_sampler: Type[Sampler] = torch.utils.data.RandomSampler, decode_audio: bool = True, decoder: str = "pyav", - **data_source_kwargs, + **data_source_kwargs: Any, ): self.clip_sampler = clip_sampler self.clip_duration = clip_duration From 4a64b11e93c81121933f8140bb22ac1e90bc8d62 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 20:01:59 -0400 Subject: [PATCH 20/54] always use a Label class --- flash/core/data/data_source.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 1a69d1de65..34d128c802 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -44,9 +44,10 @@ from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: + from fiftyone.core.labels import Label from fiftyone.core.collections import SampleCollection else: - SampleCollection = None + Label, SampleCollection = None, None # Credit to the PyTorchVision Team: @@ -481,7 +482,7 @@ def __init__(self, label_field: str = "ground_truth"): @property def label_cls(self): - return None + return Label def load_data(self, data: SampleCollection, @@ -512,9 +513,6 @@ def predict_load_data(self, return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def _validate(self, data): - if self.label_cls is None: - return - label_type = data._get_label_field_type(self.label_field) if not issubclass(label_type, self.label_cls): raise ValueError("Expected field '%s' to have type %s; found %s" % (self.label_field, self.label_cls, label_type)) From 42474d837b8b49e498433d4dca37448cba4c8903 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 20:20:45 -0400 Subject: [PATCH 21/54] exposing base class params --- flash/core/classification.py | 2 +- flash/image/detection/serialization.py | 7 ++++++- flash/image/segmentation/serialization.py | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 508f19dc1e..3da4f33c04 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -167,7 +167,7 @@ def serialize(self, sample: Any) -> Union[int, List[int], str, List[str]]: class FiftyOneLabels(ClassificationSerializer): - """A :class:`.Serializer` which converts the model outputs (assumed to be logits) to FiftyOne classification format. + """A :class:`.Serializer` which converts the model outputs to FiftyOne classification format. Args: labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 059265a206..e98a06065c 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -26,12 +26,17 @@ class FiftyOneDetectionLabels(Serializer): - """A :class:`.Serializer` which converts the model outputs to FiftyOne detection format. + """A :class:`.Serializer` which converts model outputs to FiftyOne detection format. + + Args: + labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not + provided, will attempt to get them from the :class:`.LabelsState`. """ def __init__(self, labels: Optional[List[str]] = None): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + super().__init__() self._labels = labels diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 7c09c87177..58b7b2fb5f 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -45,7 +45,7 @@ def __init__(self, labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, Args: labels_map: A dictionary that map the labels ids to pixel intensities. - visualise: Wether to visualise the image labels. + visualize: Wether to visualize the image labels. """ super().__init__() self.labels_map = labels_map @@ -89,8 +89,18 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> torch.Tensor: class FiftyOneSegmentationLabels(SegmentationLabels): + def __init__(self, labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, visualize: bool = False): + """A :class:`.Serializer` which converts the model outputs to FiftyOne segmentation format. + + Args: + labels_map: A dictionary that map the labels ids to pixel intensities. + visualize: Wether to visualize the image labels. + """ + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + + super().__init__(labels_map=labels_map, visualize=visualize) + def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: - preds = sample[DefaultDataKeys.PREDS] - assert len(preds.shape) == 3, preds.shape - labels = torch.argmax(preds, dim=-3).numpy() # HxW + labels = super().serialize(sample) return Segmentation(mask=labels) From 5b1b09c3f08bf5c4881ed945b92114297bf9e7d8 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 20:25:43 -0400 Subject: [PATCH 22/54] indent --- flash/image/segmentation/serialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 58b7b2fb5f..d759f6c38b 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -101,6 +101,6 @@ def __init__(self, labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, super().__init__(labels_map=labels_map, visualize=visualize) - def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: + def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: labels = super().serialize(sample) return Segmentation(mask=labels) From 64163b31e7958a4bf08198faf386233e0fe98d05 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 20:41:44 -0400 Subject: [PATCH 23/54] revert segmentation optimization --- flash/image/segmentation/data.py | 9 ++++----- flash/image/segmentation/serialization.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index fd746dc837..ba5c23a4ed 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -42,10 +42,11 @@ from flash.image.segmentation.transforms import default_transforms, train_default_transforms if _FIFTYONE_AVAILABLE: + import fiftyone as fo from fiftyone.core.labels import Segmentation from fiftyone.core.collections import SampleCollection else: - Segmentation, SampleCollection = None, None + fo, Segmentation, SampleCollection = None, None, None if _MATPLOTLIB_AVAILABLE: import matplotlib.pyplot as plt @@ -164,7 +165,6 @@ def __init__(self, label_field: str = "ground_truth"): raise ModuleNotFoundError("Please, pip install -e '.[image]'") super().__init__(label_field=label_field) - self._fo_dataset = None self._fo_dataset_name = None @property @@ -180,11 +180,10 @@ def load_data(self, return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: - if self._fo_dataset is None: - self._fo_dataset = fo.load_dataset(self._fo_dataset_name) + _fo_dataset = fo.load_dataset(self._fo_dataset_name) img_path = sample[DefaultDataKeys.INPUT] - fo_sample = self._fo_dataset[img_path] + fo_sample = _fo_dataset[img_path] img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index d759f6c38b..0e426f404c 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -103,4 +103,4 @@ def __init__(self, labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: labels = super().serialize(sample) - return Segmentation(mask=labels) + return Segmentation(mask=labels.numpy()) From 3763262578b6999d064d0550890fd8af07eac01e Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 20:51:34 -0400 Subject: [PATCH 24/54] revert to mutli --- flash/core/classification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 3da4f33c04..ae7d1d27c6 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -75,11 +75,11 @@ class ClassificationSerializer(Serializer): def __init__(self, multi_label: bool = False): super().__init__() - self._multi_label = multi_label + self._mutli_label = multi_label @property def multi_label(self) -> bool: - return self._multi_label + return self._mutli_label class Logits(ClassificationSerializer): From 30f0ec37b6ab88c236bb593aafca007982545ea4 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 21:07:32 -0400 Subject: [PATCH 25/54] linting --- flash/core/data/data_source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 34d128c802..1c0b7f4430 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -470,6 +470,7 @@ class NumpyDataSource(SequenceDataSource[np.ndarray]): """The ``NumpyDataSource`` is a ``SequenceDataSource`` which expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence of ``np.ndarray`` objects.""" + class FiftyOneDataSource(DataSource[SampleCollection]): """The ``FiftyOneDataSource`` expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a ``fiftyone.core.collections.SampleCollection``.""" From 9b91ea10bafff910c6a3ae15e3cb6e87913eecc2 Mon Sep 17 00:00:00 2001 From: brimoor Date: Thu, 3 Jun 2021 21:29:39 -0400 Subject: [PATCH 26/54] adding support for label thresholding --- flash/core/classification.py | 38 +++++++++++++++++--------- flash/image/detection/serialization.py | 13 ++++++--- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index ae7d1d27c6..faf99ba762 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -173,7 +173,8 @@ class FiftyOneLabels(ClassificationSerializer): labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not provided, will attempt to get them from the :class:`.LabelsState`. multi_label: If true, treats outputs as multi label logits. - threshold: The threshold to use for multi_label classification. + threshold: A threshold to use to filter candidate labels. In the single label case, predictions below this + threshold will be replaced with None store_logits: Boolean determining whether to store logits in the FiftyOne labels """ @@ -181,16 +182,19 @@ def __init__( self, labels: Optional[List[str]] = None, multi_label: bool = False, - threshold: float = 0.5, + threshold: Optional[float] = None, store_logits: bool = False, ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") + if multi_label and threshold is None: + threshold = 0.5 + super().__init__(multi_label=multi_label) + self._labels = labels self.threshold = threshold self.store_logits = store_logits - self._labels = labels if labels is not None: self.set_state(LabelsState(labels)) @@ -234,11 +238,15 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: logits=logits, ) else: - fo_labels = Classification( - label=labels[classes], - confidence=max(probabilities), - logits=logits, - ) + confidence = max(probabilities) + if self.threshold is not None and confidence < self.threshold: + fo_labels = None + else: + fo_labels = Classification( + label=labels[classes], + confidence=confidence, + logits=logits, + ) else: rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) @@ -255,10 +263,14 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: logits=logits, ) else: - fo_labels = Classification( - label=str(classes), - confidence=max(probabilities), - logits=logits, - ) + confidence = max(probabilities) + if self.threshold is not None and confidence < self.threshold: + fo_labels = None + else: + fo_labels = Classification( + label=str(classes), + confidence=confidence, + logits=logits, + ) return fo_labels diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index e98a06065c..df53574ce5 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -31,14 +31,16 @@ class FiftyOneDetectionLabels(Serializer): Args: labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not provided, will attempt to get them from the :class:`.LabelsState`. + threshold: a score threshold to apply to candidate detections. """ - def __init__(self, labels: Optional[List[str]] = None): + def __init__(self, labels: Optional[List[str]] = None, threshold: Optional[float] = None): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") super().__init__() self._labels = labels + self.threshold = threshold if labels is not None: self.set_state(LabelsState(labels)) @@ -57,6 +59,11 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: detections = [] for det in sample: + confidence = det["scores"].tolist() + + if self.threshold is not None and confidence < self.threshold: + continue + xmin, ymin, xmax, ymax = [c.tolist() for c in det["boxes"]] box = [xmin, ymin, xmax - xmin, ymax - ymin] @@ -66,13 +73,11 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: else: label = str(int(label)) - score = det["scores"].tolist() - detections.append( Detection( label=label, bounding_box=box, - confidence=score, + confidence=confidence, ) ) From 09858af108205035270c8ecfae7a3d6044e3ba06 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Thu, 3 Jun 2021 21:29:52 -0400 Subject: [PATCH 27/54] linting --- flash/core/classification.py | 12 ++++----- flash/core/data/data_source.py | 19 +++++++++----- flash/image/data.py | 8 +++++- flash/image/segmentation/data.py | 2 +- flash/video/classification/data.py | 6 ++--- tests/core/test_classification.py | 25 ++++++++++++------- .../test_data_model_integration.py | 1 + tests/image/detection/test_data.py | 14 +++++------ .../detection/test_data_model_integration.py | 1 + tests/image/detection/test_serialization.py | 2 +- 10 files changed, 56 insertions(+), 34 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index ae7d1d27c6..1b55957124 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -178,12 +178,12 @@ class FiftyOneLabels(ClassificationSerializer): """ def __init__( - self, - labels: Optional[List[str]] = None, - multi_label: bool = False, - threshold: float = 0.5, - store_logits: bool = False, - ): + self, + labels: Optional[List[str]] = None, + multi_label: bool = False, + threshold: float = 0.5, + store_logits: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 34d128c802..fdc3c0d7f6 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -225,7 +225,8 @@ def load_data(self, return data def load_sample(self, sample: Mapping[str, Any], dataset: Optional[Any] = None) -> Any: - """Given an element from the output of a call to :meth:`~flash.core.data.data_source.DataSource.load_data`, this hook + """Given an element from the output of a call to + :meth:`~flash.core.data.data_source.DataSource.load_data`, this hook should load a single data sample. The keys and values in the ``sample`` argument will be same as the keys and values in the outputs of :meth:`~flash.core.data.data_source.DataSource.load_data`. @@ -286,8 +287,8 @@ def generate_dataset( data: Optional[DATA_TYPE], running_stage: RunningStage, ) -> Optional[Union[AutoDataset, IterableAutoDataset]]: - """Generate a single dataset with the given input to :meth:`~flash.core.data.data_source.DataSource.load_data` for - the given ``running_stage``. + """Generate a single dataset with the given input to + :meth:`~flash.core.data.data_source.DataSource.load_data` for the given ``running_stage``. Args: data: The input to :meth:`~flash.core.data.data_source.DataSource.load_data` to use to create the dataset. @@ -470,6 +471,7 @@ class NumpyDataSource(SequenceDataSource[np.ndarray]): """The ``NumpyDataSource`` is a ``SequenceDataSource`` which expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a sequence of ``np.ndarray`` objects.""" + class FiftyOneDataSource(DataSource[SampleCollection]): """The ``FiftyOneDataSource`` expects the input to :meth:`~flash.core.data.data_source.DataSource.load_data` to be a ``fiftyone.core.collections.SampleCollection``.""" @@ -498,9 +500,13 @@ def load_data(self, dataset.num_classes = len(classes) class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} - to_idx = lambda t: class_to_idx[t] + if targets and isinstance(targets[0], list): - to_idx = lambda t: [class_to_idx[x] for x in t] + def to_idx(t): + return [class_to_idx[x] for x in t] + else: + def to_idx(t): + return class_to_idx[t] return [ {DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t)} @@ -515,7 +521,8 @@ def predict_load_data(self, def _validate(self, data): label_type = data._get_label_field_type(self.label_field) if not issubclass(label_type, self.label_cls): - raise ValueError("Expected field '%s' to have type %s; found %s" % (self.label_field, self.label_cls, label_type)) + raise ValueError("Expected field '%s' to have type %s; found %s" % + (self.label_field, self.label_cls, label_type)) def _get_classes(self, data): classes = data.classes.get(self.label_field, None) diff --git a/flash/image/data.py b/flash/image/data.py index e82f5af0f6..295815692a 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -15,7 +15,13 @@ import torch -from flash.core.data.data_source import DefaultDataKeys, FiftyOneDataSource, NumpyDataSource, PathsDataSource, TensorDataSource +from flash.core.data.data_source import ( + DefaultDataKeys, + FiftyOneDataSource, + NumpyDataSource, + PathsDataSource, + TensorDataSource, +) from flash.core.utilities.imports import _TORCHVISION_AVAILABLE if _TORCHVISION_AVAILABLE: diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index ba5c23a4ed..004ff1e2a4 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -186,7 +186,7 @@ def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Ten fo_sample = _fo_dataset[img_path] img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW - img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW + img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW return { DefaultDataKeys.INPUT: img.float(), diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index ed80e8fa17..db6981680e 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -169,15 +169,15 @@ def __init__( def label_cls(self): return Classification - def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodededVideoDataset': + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': self._validate(data) classes = self._get_classes(data) label_to_class_mapping = dict(enumerate(classes)) - class_to_label_mapping = {c: l for l, c in label_to_class_mapping.items()} + class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} filepaths, labels = data.values(["filepath", self.label_field + ".label"]) - targets = [class_to_label_mapping[l] for l in labels] + targets = [class_to_label_mapping[lab] for lab in labels] labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index 4c3b0cb8c3..654415b6c5 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -24,9 +24,11 @@ def test_classification_serializers(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits().serialize(example_output)), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True).serialize(example_output).logits), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels( + store_logits=True).serialize(example_output).logits), example_output) assert torch.allclose(torch.tensor(Probabilities().serialize(example_output)), torch.softmax(example_output, -1)) - assert torch.allclose(torch.tensor(FiftyOneLabels().serialize(example_output).confidence), torch.softmax(example_output, -1)[-1]) + assert torch.allclose(torch.tensor(FiftyOneLabels().serialize( + example_output).confidence), torch.softmax(example_output, -1)[-1]) assert Classes().serialize(example_output) == 2 assert Labels(labels).serialize(example_output) == 'class_3' assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' @@ -38,7 +40,8 @@ def test_classification_serializers_multi_label(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits(multi_label=True).serialize(example_output)), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True,multi_label=True).serialize(example_output).logits), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels( + store_logits=True, multi_label=True).serialize(example_output).logits), example_output) assert torch.allclose( torch.tensor(Probabilities(multi_label=True).serialize(example_output)), torch.sigmoid(example_output), @@ -46,7 +49,8 @@ def test_classification_serializers_multi_label(): assert Classes(multi_label=True).serialize(example_output) == [1, 2] assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] assert Labels(labels, multi_label=True).serialize(example_output) == ['class_2', 'class_3'] - assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize(example_output).classifications] == ['class_2', 'class_3'] + assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize( + example_output).classifications] == ['class_2', 'class_3'] @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") @@ -54,11 +58,14 @@ def test_classification_serializers_fiftyone(): example_output = torch.tensor([-0.1, 0.2, 0.3]) # 3 classes labels = ['class_1', 'class_2', 'class_3'] - assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True).serialize(example_output).logits), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels().serialize(example_output).confidence), torch.softmax(example_output, -1)[-1]) + assert torch.allclose(torch.tensor(FiftyOneLabels( + store_logits=True).serialize(example_output).logits), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels().serialize( + example_output).confidence), torch.softmax(example_output, -1)[-1]) assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' assert FiftyOneLabels().serialize(example_output).label == '2' - assert torch.allclose(torch.tensor(FiftyOneLabels(store_logits=True,multi_label=True).serialize(example_output).logits), example_output) + assert torch.allclose(torch.tensor(FiftyOneLabels( + store_logits=True, multi_label=True).serialize(example_output).logits), example_output) assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] - assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize(example_output).classifications] == ['class_2', 'class_3'] - + assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize( + example_output).classifications] == ['class_2', 'class_3'] diff --git a/tests/image/classification/test_data_model_integration.py b/tests/image/classification/test_data_model_integration.py index 9414c35f8e..711bcc329f 100644 --- a/tests/image/classification/test_data_model_integration.py +++ b/tests/image/classification/test_data_model_integration.py @@ -60,6 +60,7 @@ def test_classification(tmpdir): trainer = Trainer(default_root_dir=tmpdir, fast_dev_run=True) trainer.finetune(model, datamodule=data, strategy="freeze") + @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed.") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") def test_classification_fiftyone(tmpdir): diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index 1f132f1c3f..d98e02ee2e 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -95,16 +95,16 @@ def _create_synth_fiftyone_dataset(tmpdir): sample2 = dataset[str(img_dir / "sample_two.png")] d1 = fo.Detection( - label = "person", - bounding_box = [0.3, 0.4, 0.2, 0.2], + label="person", + bounding_box=[0.3, 0.4, 0.2, 0.2], ) d2 = fo.Detection( - label = "person", - bounding_box = [0.05, 0.10, 0.28, 0.15], + label="person", + bounding_box=[0.05, 0.10, 0.28, 0.15], ) d3 = fo.Detection( - label = "person", - bounding_box = [0.23, 0.14, 0.09, 0.18], + label="person", + bounding_box=[0.23, 0.14, 0.09, 0.18], ) d1["iscrowd"] = 1 d2["iscrowd"] = 0 @@ -114,7 +114,7 @@ def _create_synth_fiftyone_dataset(tmpdir): detections=[d1] ) sample2["ground_truth"] = fo.Detections( - detections=[d2,d3] + detections=[d2, d3] ) sample1.save() diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index 89be010a95..a20a4c06d3 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -55,6 +55,7 @@ def test_detection(tmpdir, model, backbone): test_images = [str(test_image_one), str(test_image_two)] model.predict(test_images) + @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed for testing") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") @pytest.mark.parametrize(["model", "backbone"], [("fasterrcnn", "resnet18")]) diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py index 9da284c628..cdaa28b3f9 100644 --- a/tests/image/detection/test_serialization.py +++ b/tests/image/detection/test_serialization.py @@ -23,4 +23,4 @@ def test_serialize_fiftyone(self): detections = serial.serialize(sample) assert len(detections.detections) == 1 - assert detections.detections[0].bounding_box == [20,30,20,20] + assert detections.detections[0].bounding_box == [20, 30, 20, 20] From 0ce6ede57ebc4b39fc1819e777cacbe8913128fc Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Fri, 4 Jun 2021 00:09:12 -0400 Subject: [PATCH 28/54] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad722ea00..f429b45e0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [0.3.1] - YYYY-MM-DD +### Added + +- Added integration with FiftyOne ([#360](https://github.com/PyTorchLightning/lightning-flash/pull/360)) + ### Fixed - Fixed `flash.Trainer.add_argparse_args` not adding any arguments ([#343](https://github.com/PyTorchLightning/lightning-flash/pull/343)) From 5278af2fe24c3ee5b9b181cde74c9855489a71d9 Mon Sep 17 00:00:00 2001 From: tchaton Date: Fri, 4 Jun 2021 14:28:56 +0100 Subject: [PATCH 29/54] resolve some issues, clean API --- flash/core/classification.py | 40 +++-- flash/core/data/data_module.py | 139 +++++++++++++++++- flash/core/data/data_source.py | 36 ++--- flash/core/integrations/__init__.py | 0 flash/core/integrations/fiftyone/__init__.py | 1 + flash/core/integrations/fiftyone/utils.py | 40 +++++ flash/image/classification/model.py | 6 +- flash/image/detection/data.py | 22 +-- flash/video/classification/data.py | 50 +++---- .../finetuning/semantic_segmentation.py | 3 + .../fiftyone/image_classification.py | 31 ++++ requirements/datatype_image.txt | 1 + requirements/datatype_video.txt | 1 + tests/core/test_classification.py | 43 +++--- tests/core/test_integrations.py | 40 +++++ tests/examples/test_integrations.py | 40 +++++ tests/examples/test_scripts.py | 39 +---- tests/examples/utils.py | 57 +++++++ tests/image/classification/test_data.py | 4 +- .../test_data_model_integration.py | 2 +- tests/image/detection/test_data.py | 17 +-- .../detection/test_data_model_integration.py | 2 +- tests/image/segmentation/test_data.py | 4 +- tests/video/test_video_classifier.py | 6 +- 24 files changed, 466 insertions(+), 158 deletions(-) create mode 100644 flash/core/integrations/__init__.py create mode 100644 flash/core/integrations/fiftyone/__init__.py create mode 100644 flash/core/integrations/fiftyone/utils.py create mode 100644 flash_examples_integrations/fiftyone/image_classification.py create mode 100644 tests/core/test_integrations.py create mode 100644 tests/examples/test_integrations.py create mode 100644 tests/examples/utils.py diff --git a/flash/core/classification.py b/flash/core/classification.py index 3adfd2cb96..84c1c572d8 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -11,19 +11,20 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, List, Mapping, Optional, Sequence, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Union import torch import torch.nn.functional as F import torchmetrics from pytorch_lightning.utilities import rank_zero_warn -from flash.core.data.data_source import LabelsState +from flash.core.data.data_source import DefaultDataKeys, LabelsState from flash.core.data.process import Serializer from flash.core.model import Task from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: + import fiftyone as fo from fiftyone.core.labels import Classification, Classifications else: Classification, Classifications = None, None @@ -86,6 +87,8 @@ class Logits(ClassificationSerializer): """A :class:`.Serializer` which simply converts the model outputs (assumed to be logits) to a list.""" def serialize(self, sample: Any) -> Any: + sample = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + sample = torch.tensor(sample) return sample.tolist() @@ -94,6 +97,8 @@ class Probabilities(ClassificationSerializer): list.""" def serialize(self, sample: Any) -> Any: + sample = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + sample = torch.tensor(sample) if self.multi_label: return torch.sigmoid(sample).tolist() return torch.softmax(sample, -1).tolist() @@ -115,6 +120,8 @@ def __init__(self, multi_label: bool = False, threshold: float = 0.5): self.threshold = threshold def serialize(self, sample: Any) -> Union[int, List[int]]: + sample = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + sample = torch.tensor(sample) if self.multi_label: one_hot = (sample.sigmoid() > self.threshold).int().tolist() result = [] @@ -146,6 +153,8 @@ def __init__(self, labels: Optional[List[str]] = None, multi_label: bool = False self.set_state(LabelsState(labels)) def serialize(self, sample: Any) -> Union[int, List[int], str, List[str]]: + sample = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + sample = torch.tensor(sample) labels = None if self._labels is not None: @@ -200,6 +209,9 @@ def __init__( self.set_state(LabelsState(labels)) def serialize(self, sample: Any) -> Union[Classification, Classifications]: + pred = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + pred = torch.tensor(pred) + metadata = sample[DefaultDataKeys.METADATA] labels = None if self._labels is not None: @@ -211,18 +223,18 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: logits = None if self.store_logits: - logits = sample.tolist() + logits = pred.tolist() if self.multi_label: - one_hot = (sample.sigmoid() > self.threshold).int().tolist() + one_hot = (pred.sigmoid() > self.threshold).int().tolist() classes = [] for index, value in enumerate(one_hot): if value == 1: classes.append(index) - probabilities = torch.sigmoid(sample).tolist() + probabilities = torch.sigmoid(pred).tolist() else: - classes = torch.argmax(sample, -1).tolist() - probabilities = torch.softmax(sample, -1).tolist() + classes = torch.argmax(pred, -1).tolist() + probabilities = torch.softmax(pred, -1).tolist() if labels is not None: if self.multi_label: @@ -233,16 +245,16 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: confidence=probabilities[idx], ) classifications.append(fo_cls) - fo_labels = Classifications( + fo_predictions = Classifications( classifications=classifications, logits=logits, ) else: confidence = max(probabilities) if self.threshold is not None and confidence < self.threshold: - fo_labels = None + fo_predictions = None else: - fo_labels = Classification( + fo_predictions = Classification( label=labels[classes], confidence=confidence, logits=logits, @@ -258,19 +270,19 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: confidence=probabilities[idx], ) classifications.append(fo_cls) - fo_labels = Classifications( + fo_predictions = Classifications( classifications=classifications, logits=logits, ) else: confidence = max(probabilities) if self.threshold is not None and confidence < self.threshold: - fo_labels = None + fo_predictions = None else: - fo_labels = Classification( + fo_predictions = Classification( label=str(classes), confidence=confidence, logits=logits, ) - return fo_labels + return fo.Sample(filepath=metadata.filepath, predictions=fo_predictions) diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 3b1937a2c7..327896123a 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -33,9 +33,12 @@ from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: + import fiftyone as fo from fiftyone.core.collections import SampleCollection + from fiftyone.types import Dataset else: SampleCollection = None + Dataset = None class DataModule(pl.LightningDataModule): @@ -1032,7 +1035,7 @@ def from_datasets( ) @classmethod - def from_fiftyone( + def fiftyone_from_datasets( cls, train_dataset: Optional[SampleCollection] = None, val_dataset: Optional[SampleCollection] = None, @@ -1088,13 +1091,16 @@ def from_fiftyone( "/path/to/dataset", dataset_type=fo.types.ImageClassificationDirectoryTree, ) - data_module = DataModule.from_fiftyone( + data_module = DataModule.fiftyone_from_datasets( train_data = train_dataset, train_transform={ "to_tensor_transform": torch.as_tensor, }, ) """ + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, `pip install fiftyone`.") + return cls.from_data_source( DefaultDataSources.FIFTYONE, train_dataset, @@ -1112,3 +1118,132 @@ def from_fiftyone( num_workers=num_workers, **preprocess_kwargs, ) + + @classmethod + def fiftyone_from_dir( + cls, + train_dir: Optional[SampleCollection] = None, + val_dir: Optional[SampleCollection] = None, + test_dir: Optional[SampleCollection] = None, + predict_dir: Optional[SampleCollection] = None, + train_transform: Optional[Dict[str, Callable]] = None, + val_transform: Optional[Dict[str, Callable]] = None, + test_transform: Optional[Dict[str, Callable]] = None, + predict_transform: Optional[Dict[str, Callable]] = None, + data_fetcher: Optional[BaseDataFetcher] = None, + preprocess: Optional[Preprocess] = None, + val_split: Optional[float] = None, + batch_size: int = 4, + num_workers: Optional[int] = None, + dataset_type: Optional[Dataset] = None, + predict_dataset_type: Optional[Dataset] = None, + label_field: str = 'ground_truth', + tags: List[str] = None, + importer_kwargs: Dict[str, Any] = {}, + predict_importer_kwargs: Dict[str, Any] = {}, + **preprocess_kwargs: Any, + ) -> 'DataModule': + """Creates a :class:`~flash.core.data.data_module.DataModule` object + from the given FiftyOne Datasets using the + :class:`~flash.core.data.data_source.DataSource` of name + :attr:`~flash.core.data.data_source.DefaultDataSources.FIFTYONE` + from the passed or constructed :class:`~flash.core.data.process.Preprocess`. + + Args: + train_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the train data. + val_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the validation data. + test_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the test data. + predict_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the predict data. + train_transform: The dictionary of transforms to use during training which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + val_transform: The dictionary of transforms to use during validation which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + test_transform: The dictionary of transforms to use during testing which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + predict_transform: The dictionary of transforms to use during predicting which maps + :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. + data_fetcher: The :class:`~flash.core.data.callback.BaseDataFetcher` to pass to the + :class:`~flash.core.data.data_module.DataModule`. + preprocess: The :class:`~flash.core.data.data.Preprocess` to pass to the + :class:`~flash.core.data.data_module.DataModule`. If ``None``, ``cls.preprocess_cls`` + will be constructed and used. + val_split: The ``val_split`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + batch_size: The ``batch_size`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + num_workers: The ``num_workers`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. + preprocess_kwargs: Additional keyword arguments to use when constructing the preprocess. Will only be used + if ``preprocess = None``. + + Returns: + The constructed data module. + + Examples:: + + train_dataset = fo.Dataset.from_dir( + "/path/to/dataset", + dataset_type=fo.types.ImageClassificationDirectoryTree, + ) + data_module = DataModule.fiftyone_from_datasets( + train_data = train_dataset, + train_transform={ + "to_tensor_transform": torch.as_tensor, + }, + ) + """ + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, `pip install fiftyone`.") + + if not dataset_type: + dataset_type = fo.types.ImageClassificationDirectoryTree + + if not predict_dataset_type: + predict_dataset_type = fo.types.ImageDirectory + + train_dataset = fo.Dataset.from_dir( + train_dir, + dataset_type, + label_field=label_field, + tags=tags, + **importer_kwargs, + ) + + val_dataset = fo.Dataset.from_dir( + val_dir, + dataset_type, + label_field=label_field, + tags=tags, + **importer_kwargs, + ) + + test_dataset = fo.Dataset.from_dir( + test_dir, + dataset_type, + label_field=label_field, + tags=tags, + **importer_kwargs, + ) + + predict_dataset = fo.Dataset.from_dir( + predict_dir, + predict_dataset_type, + label_field=label_field, + tags=tags, + **predict_importer_kwargs, + ) + + return cls.fiftyone_from_datasets( + train_dataset=train_dataset, + val_dataset=val_dataset, + test_dataset=test_dataset, + predict_dataset=predict_dataset, + train_transform=train_transform, + val_transform=val_transform, + test_transform=test_transform, + predict_transform=predict_transform, + data_fetcher=data_fetcher, + preprocess=preprocess, + val_split=val_split, + batch_size=batch_size, + num_workers=num_workers, + label_field=label_field, + **preprocess_kwargs, + ) diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index fdc3c0d7f6..6ba58240f8 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -44,8 +44,8 @@ from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: - from fiftyone.core.labels import Label from fiftyone.core.collections import SampleCollection + from fiftyone.core.labels import Label else: Label, SampleCollection = None, None @@ -486,13 +486,13 @@ def __init__(self, label_field: str = "ground_truth"): def label_cls(self): return Label - def load_data(self, - data: SampleCollection, - dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: self._validate(data) - _, label_path = data._get_label_field_path(self.label_field, "label") - filepaths, targets = data.values(["filepath", label_path]) + label_path = data._get_label_field_path(self.label_field, "label")[1] + + filepaths = data.values("filepath") + targets = data.values(label_path) classes = self._get_classes(data) @@ -502,27 +502,29 @@ def load_data(self, class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} if targets and isinstance(targets[0], list): + def to_idx(t): return [class_to_idx[x] for x in t] else: + def to_idx(t): return class_to_idx[t] - return [ - {DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t)} - for f, t in zip(filepaths, targets) - ] + return [{ + DefaultDataKeys.INPUT: f, + DefaultDataKeys.TARGET: to_idx(t), + DefaultDataKeys.METADATA: data[f] + } for f, t in zip(filepaths, targets)] - def predict_load_data(self, - data: SampleCollection, - dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] + def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.METADATA: data[f]} for f in data.values("filepath")] def _validate(self, data): label_type = data._get_label_field_type(self.label_field) if not issubclass(label_type, self.label_cls): - raise ValueError("Expected field '%s' to have type %s; found %s" % - (self.label_field, self.label_cls, label_type)) + raise ValueError( + "Expected field '%s' to have type %s; found %s" % (self.label_field, self.label_cls, label_type) + ) def _get_classes(self, data): classes = data.classes.get(self.label_field, None) @@ -531,7 +533,7 @@ def _get_classes(self, data): classes = data.default_classes if not classes: - _, label_path = data._get_label_field_path(self.label_field, "label") + label_path = data._get_label_field_path(self.label_field, "label")[1] classes = data.distinct(label_path) return classes diff --git a/flash/core/integrations/__init__.py b/flash/core/integrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/flash/core/integrations/fiftyone/__init__.py b/flash/core/integrations/fiftyone/__init__.py new file mode 100644 index 0000000000..17e27252e6 --- /dev/null +++ b/flash/core/integrations/fiftyone/__init__.py @@ -0,0 +1 @@ +from flash.core.integrations.fiftyone.utils import fiftyone_launch_app, get_classes diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py new file mode 100644 index 0000000000..042433a764 --- /dev/null +++ b/flash/core/integrations/fiftyone/utils.py @@ -0,0 +1,40 @@ +from time import sleep as sleep_fn +from typing import List, Optional + +import flash +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE + +if _FIFTYONE_AVAILABLE: + import fiftyone as fo + from fiftyone import Sample + from fiftyone.core.session import Session +else: + Sample = None + Session = None + + +def fiftyone_launch_app(samples: List[Sample], sleep: Optional[int] = 120, **kwargs) -> Optional[Session]: + if not _FIFTYONE_AVAILABLE: + raise ModuleNotFoundError("Please, `pip install fiftyone`.") + if flash._IS_TESTING: + return None + dataset = fo.Dataset() + for sample in samples: + dataset.add_samples(sample) + session = fo.launch_app(dataset, **kwargs) + if sleep: + sleep_fn(sleep) + return session + + +def get_classes(data, label_field: str): + classes = data.classes.get(label_field, None) + + if not classes: + classes = data.default_classes + + if not classes: + label_path = data._get_label_field_path(label_field, "label")[1] + classes = data.distinct(label_path) + + return classes diff --git a/flash/image/classification/model.py b/flash/image/classification/model.py index e172e5fe86..0c2d8d4c16 100644 --- a/flash/image/classification/model.py +++ b/flash/image/classification/model.py @@ -123,8 +123,10 @@ def test_step(self, batch: Any, batch_idx: int) -> Any: return super().test_step(batch, batch_idx) def predict_step(self, batch: Any, batch_idx: int, dataloader_idx: int = 0) -> Any: - batch = (batch[DefaultDataKeys.INPUT]) - return super().predict_step(batch, batch_idx, dataloader_idx=dataloader_idx) + batch[DefaultDataKeys.PREDS] = super().predict_step((batch[DefaultDataKeys.INPUT]), + batch_idx, + dataloader_idx=dataloader_idx) + return batch def forward(self, x) -> torch.Tensor: x = self.backbone(x) diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index a2b7940a14..7c1ca5abb0 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -26,8 +26,8 @@ from pycocotools.coco import COCO if _FIFTYONE_AVAILABLE: - from fiftyone.core.labels import Detections from fiftyone.core.collections import SampleCollection + from fiftyone.core.labels import Detections else: Detections, SampleCollection = None, None @@ -106,23 +106,17 @@ def __init__(self, label_field: str = "ground_truth", iscrowd: str = "iscrowd"): def label_cls(self): return Detections - def load_data(self, - data: SampleCollection, - dataset: Optional[Any] = None) -> Sequence[Dict[str, Any]]: + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Dict[str, Any]]: self._validate(data) data.compute_metadata() - filepaths, widths, heights, labels, bboxes, iscrowds = data.values( - [ - "filepath", - "metadata.width", - "metadata.height", - self.label_field + ".detections.label", - self.label_field + ".detections.bounding_box", - self.label_field + ".detections." + self.iscrowd, - ] - ) + filepaths = data.values("filepath") + widths = data.values("metadata.width") + heights = data.values("metadata.height") + labels = data.values(self.label_field + ".detections.label") + bboxes = data.values(self.label_field + ".detections.bounding_box") + iscrowds = data.values(self.label_field + ".detections." + self.iscrowd) classes = self._get_classes(data) class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index db6981680e..c4af6c13b3 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -29,15 +29,13 @@ ) from flash.core.data.process import Preprocess from flash.core.data.transforms import merge_transforms -from flash.core.utilities.imports import ( - _FIFTYONE_AVAILABLE, - _KORNIA_AVAILABLE, - _PYTORCHVIDEO_AVAILABLE, -) +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE, _KORNIA_AVAILABLE, _PYTORCHVIDEO_AVAILABLE if _FIFTYONE_AVAILABLE: - from fiftyone.core.labels import Classification from fiftyone.core.collections import SampleCollection + from fiftyone.core.labels import Classification + + from flash.core.integrations.fiftyone import get_classes else: Classification, SampleCollection = None, None @@ -62,7 +60,7 @@ _PYTORCHVIDEO_DATA = Dict[str, Union[str, torch.Tensor, int, float, List]] -class _VideoClassificationMixin(object): +class BaseVideoClassification(object): def __init__( self, @@ -110,7 +108,7 @@ def _encoded_video_to_dict(self, video) -> Dict[str, Any]: } -class VideoClassificationPathsDataSource(PathsDataSource, _VideoClassificationMixin): +class VideoClassificationPathsDataSource(PathsDataSource, BaseVideoClassification): def __init__( self, @@ -120,7 +118,7 @@ def __init__( decoder: str = "pyav", ): super().__init__(extensions=("mp4", "avi")) - _VideoClassificationMixin.__init__( + BaseVideoClassification.__init__( self, clip_sampler, video_sampler=video_sampler, @@ -128,7 +126,7 @@ def __init__( decoder=decoder, ) - def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': + def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': ds: EncodedVideoDataset = labeled_encoded_video_dataset( pathlib.Path(data), self.clip_sampler, @@ -136,6 +134,10 @@ def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDa decode_audio=self.decode_audio, decoder=self.decoder, ) + return ds + + def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': + ds = self._make_encoded_video_dataset(data) if self.training: label_to_class_mapping = {p[1]: p[0].split("/")[-2] for p in ds._labeled_videos._paths_and_labels} self.set_state(LabelsState(label_to_class_mapping)) @@ -146,7 +148,7 @@ def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) -class VideoClassificationFiftyOneDataSource(FiftyOneDataSource, _VideoClassificationMixin): +class VideoClassificationFiftyOneDataSource(VideoClassificationPathsDataSource): def __init__( self, @@ -155,31 +157,29 @@ def __init__( decode_audio: bool = True, decoder: str = "pyav", label_field: str = "ground_truth", + **kwargs ): - super().__init__(label_field=label_field) - _VideoClassificationMixin.__init__( - self, - clip_sampler, + super().__init__( + clip_sampler=clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, decoder=decoder, + **kwargs ) + self.label_field = label_field @property def label_cls(self): return Classification - def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': - self._validate(data) - - classes = self._get_classes(data) + def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDataset': + classes = get_classes(data, self.label_field) label_to_class_mapping = dict(enumerate(classes)) class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} - filepaths, labels = data.values(["filepath", self.label_field + ".label"]) + filepaths, labels = data.values("filepath"), data.values(self.label_field + ".label") targets = [class_to_label_mapping[lab] for lab in labels] - - labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) + labeled_video_paths = LabeledVideoPaths(list(zip(data.values("filepath"), targets))) ds: EncodedVideoDataset = EncodedVideoDataset( labeled_video_paths, @@ -188,14 +188,8 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> 'E decode_audio=self.decode_audio, decoder=self.decoder, ) - if self.training: - self.set_state(LabelsState(label_to_class_mapping)) - dataset.num_classes = len(classes) return ds - def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) - class VideoClassificationPreprocess(Preprocess): diff --git a/flash_examples/finetuning/semantic_segmentation.py b/flash_examples/finetuning/semantic_segmentation.py index 2a796d9cd8..fb592d846a 100644 --- a/flash_examples/finetuning/semantic_segmentation.py +++ b/flash_examples/finetuning/semantic_segmentation.py @@ -1,3 +1,6 @@ +import pytorch_lightning as pl + +pl.seed_everything(42) # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples_integrations/fiftyone/image_classification.py new file mode 100644 index 0000000000..fdc8682e87 --- /dev/null +++ b/flash_examples_integrations/fiftyone/image_classification.py @@ -0,0 +1,31 @@ +import flash +from flash.core.classification import FiftyOneLabels, Labels, Probabilities +from flash.core.data.utils import download_data +from flash.core.finetuning import FreezeUnfreeze +from flash.core.integrations.fiftyone import fiftyone_launch_app +from flash.image import ImageClassificationData, ImageClassifier + +# 1 Download data +download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") + +# 2 Load data using fiftyone +datamodule = ImageClassificationData.fiftyone_from_dir( + train_dir="data/hymenoptera_data/train/", + val_dir="data/hymenoptera_data/val/", + test_dir="data/hymenoptera_data/test/", + predict_dir="data/hymenoptera_data/predict/", +) + +# 3 Fine tune a model +model = ImageClassifier(backbone="resnet18", num_classes=datamodule.num_classes, serializer=Labels()) +trainer = flash.Trainer(max_epochs=1, limit_train_batches=1, limit_val_batches=1) +trainer.finetune(model, datamodule=datamodule, strategy=FreezeUnfreeze(unfreeze_epoch=1)) +trainer.save_checkpoint("image_classification_model.pt") + +# 4 Predict from checkpoint +model = ImageClassifier.load_from_checkpoint("https://flash-weights.s3.amazonaws.com/image_classification_model.pt") +model.serializer = FiftyOneLabels() +predictions = trainer.predict(model, datamodule=datamodule) + +# 5. Visualize predictions in FiftyOne for 2 minutes. +session = fiftyone_launch_app(predictions) diff --git a/requirements/datatype_image.txt b/requirements/datatype_image.txt index c3e9f4ec54..21333dcd19 100644 --- a/requirements/datatype_image.txt +++ b/requirements/datatype_image.txt @@ -5,3 +5,4 @@ Pillow>=7.2 kornia>=0.5.1 matplotlib pycocotools>=2.0.2 ; python_version >= "3.7" +fiftyone diff --git a/requirements/datatype_video.txt b/requirements/datatype_video.txt index 7b7b96807e..b9bddb01ec 100644 --- a/requirements/datatype_video.txt +++ b/requirements/datatype_video.txt @@ -2,3 +2,4 @@ torchvision Pillow>=7.2 kornia>=0.5.1 pytorchvideo==0.1.0 +fiftyone diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index 654415b6c5..7a6f05fcb2 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import pytest - import torch from flash.core.classification import Classes, FiftyOneLabels, Labels, Logits, Probabilities +from flash.core.data.data_source import DefaultDataKeys from flash.core.utilities.imports import _FIFTYONE_AVAILABLE @@ -24,15 +24,9 @@ def test_classification_serializers(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits().serialize(example_output)), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels( - store_logits=True).serialize(example_output).logits), example_output) assert torch.allclose(torch.tensor(Probabilities().serialize(example_output)), torch.softmax(example_output, -1)) - assert torch.allclose(torch.tensor(FiftyOneLabels().serialize( - example_output).confidence), torch.softmax(example_output, -1)[-1]) assert Classes().serialize(example_output) == 2 assert Labels(labels).serialize(example_output) == 'class_3' - assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' - assert FiftyOneLabels().serialize(example_output).label == '2' def test_classification_serializers_multi_label(): @@ -40,32 +34,33 @@ def test_classification_serializers_multi_label(): labels = ['class_1', 'class_2', 'class_3'] assert torch.allclose(torch.tensor(Logits(multi_label=True).serialize(example_output)), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels( - store_logits=True, multi_label=True).serialize(example_output).logits), example_output) assert torch.allclose( torch.tensor(Probabilities(multi_label=True).serialize(example_output)), torch.sigmoid(example_output), ) assert Classes(multi_label=True).serialize(example_output) == [1, 2] - assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] assert Labels(labels, multi_label=True).serialize(example_output) == ['class_2', 'class_3'] - assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize( - example_output).classifications] == ['class_2', 'class_3'] @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") def test_classification_serializers_fiftyone(): - example_output = torch.tensor([-0.1, 0.2, 0.3]) # 3 classes + + class MockSample(object): + filepath = "something" + + logits = torch.tensor([-0.1, 0.2, 0.3]) + example_output = {DefaultDataKeys.PREDS: logits, DefaultDataKeys.METADATA: MockSample()} # 3 classes labels = ['class_1', 'class_2', 'class_3'] - assert torch.allclose(torch.tensor(FiftyOneLabels( - store_logits=True).serialize(example_output).logits), example_output) - assert torch.allclose(torch.tensor(FiftyOneLabels().serialize( - example_output).confidence), torch.softmax(example_output, -1)[-1]) - assert FiftyOneLabels(labels).serialize(example_output).label == 'class_3' - assert FiftyOneLabels().serialize(example_output).label == '2' - assert torch.allclose(torch.tensor(FiftyOneLabels( - store_logits=True, multi_label=True).serialize(example_output).logits), example_output) - assert [c.label for c in FiftyOneLabels(multi_label=True).serialize(example_output).classifications] == ['1', '2'] - assert [c.label for c in FiftyOneLabels(labels, multi_label=True).serialize( - example_output).classifications] == ['class_2', 'class_3'] + predictions = FiftyOneLabels(store_logits=True).serialize(example_output).predictions + assert torch.allclose(torch.tensor(predictions.logits), logits) + assert torch.allclose(torch.tensor(predictions.confidence), torch.softmax(logits, -1)[-1]) + assert predictions.label == '2' + predictions = FiftyOneLabels(labels, store_logits=True).serialize(example_output).predictions + assert predictions.label == 'class_3' + + predictions = FiftyOneLabels(store_logits=True, multi_label=True).serialize(example_output).predictions + assert torch.allclose(torch.tensor(predictions.logits), logits) + assert [c.label for c in predictions.classifications] == ['1', '2'] + predictions = FiftyOneLabels(labels, multi_label=True).serialize(example_output).predictions + assert [c.label for c in predictions.classifications] == ['class_2', 'class_3'] diff --git a/tests/core/test_integrations.py b/tests/core/test_integrations.py new file mode 100644 index 0000000000..364de1ddf1 --- /dev/null +++ b/tests/core/test_integrations.py @@ -0,0 +1,40 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional, Tuple +from unittest import mock + +import pytest + +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE +from tests.examples.utils import run_test + +root = Path(__file__).parent.parent.parent + + +@mock.patch.dict(os.environ, {"FLASH_TESTING": "1"}) +@pytest.mark.parametrize( + "folder, file", [ + pytest.param( + "fiftyone", + "image_classification.py", + marks=pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone library isn't installed") + ), + ] +) +def test_integrations(tmpdir, folder, file): + run_test(str(root / "flash_examples_integrations" / folder / file)) diff --git a/tests/examples/test_integrations.py b/tests/examples/test_integrations.py new file mode 100644 index 0000000000..364de1ddf1 --- /dev/null +++ b/tests/examples/test_integrations.py @@ -0,0 +1,40 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional, Tuple +from unittest import mock + +import pytest + +from flash.core.utilities.imports import _FIFTYONE_AVAILABLE +from tests.examples.utils import run_test + +root = Path(__file__).parent.parent.parent + + +@mock.patch.dict(os.environ, {"FLASH_TESTING": "1"}) +@pytest.mark.parametrize( + "folder, file", [ + pytest.param( + "fiftyone", + "image_classification.py", + marks=pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone library isn't installed") + ), + ] +) +def test_integrations(tmpdir, folder, file): + run_test(str(root / "flash_examples_integrations" / folder / file)) diff --git a/tests/examples/test_scripts.py b/tests/examples/test_scripts.py index cbd6eaeb90..99590e3412 100644 --- a/tests/examples/test_scripts.py +++ b/tests/examples/test_scripts.py @@ -29,50 +29,13 @@ _TORCHVISION_GREATER_EQUAL_0_9, _VIDEO_AVAILABLE, ) +from tests.examples.utils import run_test _IMAGE_AVAILABLE = _IMAGE_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_9 root = Path(__file__).parent.parent.parent -def call_script( - filepath: str, - args: Optional[List[str]] = None, - timeout: Optional[int] = 60 * 5, -) -> Tuple[int, str, str]: - with open(filepath, 'r') as original: - data = original.read() - - with open(filepath, 'w') as modified: - modified.write("import pytorch_lightning as pl\npl.seed_everything(42)\n" + data) - - if args is None: - args = [] - args = [str(a) for a in args] - command = [sys.executable, "-m", "coverage", "run", filepath] + args - print(" ".join(command)) - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - stdout, stderr = p.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - p.kill() - stdout, stderr = p.communicate() - stdout = stdout.decode("utf-8") - stderr = stderr.decode("utf-8") - - with open(filepath, 'w') as modified: - modified.write(data) - - return p.returncode, stdout, stderr - - -def run_test(filepath): - code, stdout, stderr = call_script(filepath) - print(f"{filepath} STDOUT: {stdout}") - print(f"{filepath} STDERR: {stderr}") - assert not code - - @mock.patch.dict(os.environ, {"FLASH_TESTING": "1"}) @pytest.mark.parametrize( "folder, file", diff --git a/tests/examples/utils.py b/tests/examples/utils.py new file mode 100644 index 0000000000..9cf5a4e765 --- /dev/null +++ b/tests/examples/utils.py @@ -0,0 +1,57 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional, Tuple +from unittest import mock + + +def call_script( + filepath: str, + args: Optional[List[str]] = None, + timeout: Optional[int] = 60 * 5, +) -> Tuple[int, str, str]: + with open(filepath, 'r') as original: + data = original.read() + + with open(filepath, 'w') as modified: + modified.write("import pytorch_lightning as pl\npl.seed_everything(42)\n" + data) + + if args is None: + args = [] + args = [str(a) for a in args] + command = [sys.executable, "-m", "coverage", "run", filepath] + args + print(" ".join(command)) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + stdout, stderr = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + stdout, stderr = p.communicate() + stdout = stdout.decode("utf-8") + stderr = stderr.decode("utf-8") + + with open(filepath, 'w') as modified: + modified.write(data) + + return p.returncode, stdout, stderr + + +def run_test(filepath): + code, stdout, stderr = call_script(filepath) + print(f"{filepath} STDOUT: {stdout}") + print(f"{filepath} STDERR: {stderr}") + assert not code diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index adb2f493a8..f87a4bc398 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -387,7 +387,7 @@ def test_from_data(data, from_function): @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") -def test_from_fiftyone(tmpdir): +def test_fiftyone_from_datasets(tmpdir): tmpdir = Path(tmpdir) (tmpdir / "a").mkdir() @@ -408,7 +408,7 @@ def test_from_fiftyone(tmpdir): s1.save() s2.save() - img_data = ImageClassificationData.from_fiftyone( + img_data = ImageClassificationData.fiftyone_from_datasets( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/classification/test_data_model_integration.py b/tests/image/classification/test_data_model_integration.py index 711bcc329f..68378b8fe1 100644 --- a/tests/image/classification/test_data_model_integration.py +++ b/tests/image/classification/test_data_model_integration.py @@ -84,7 +84,7 @@ def test_classification_fiftyone(tmpdir): s1.save() s2.save() - data = ImageClassificationData.from_fiftyone( + data = ImageClassificationData.fiftyone_from_datasets( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index d98e02ee2e..63310555e2 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -88,7 +88,8 @@ def _create_synth_fiftyone_dataset(tmpdir): Image.new('RGB', (1920, 1080)).save(img_dir / "sample_two.png") dataset = fo.Dataset.from_dir( - img_dir, dataset_type=fo.types.ImageDirectory, + img_dir, + dataset_type=fo.types.ImageDirectory, ) sample1 = dataset[str(img_dir / "sample_one.png")] @@ -110,12 +111,8 @@ def _create_synth_fiftyone_dataset(tmpdir): d2["iscrowd"] = 0 d3["iscrowd"] = 0 - sample1["ground_truth"] = fo.Detections( - detections=[d1] - ) - sample2["ground_truth"] = fo.Detections( - detections=[d2, d3] - ) + sample1["ground_truth"] = fo.Detections(detections=[d1]) + sample2["ground_truth"] = fo.Detections(detections=[d2, d3]) sample1.save() sample2.save() @@ -171,11 +168,11 @@ def test_image_detector_data_from_coco(tmpdir): @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") -def test_image_detector_data_from_fiftyone(tmpdir): +def test_image_detector_data_fiftyone_from_datasets(tmpdir): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - datamodule = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) + datamodule = ObjectDetectionData.fiftyone_from_datasets(train_dataset=train_dataset, batch_size=1) data = next(iter(datamodule.train_dataloader())) imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] @@ -188,7 +185,7 @@ def test_image_detector_data_from_fiftyone(tmpdir): assert datamodule.val_dataloader() is None assert datamodule.test_dataloader() is None - datamodule = ObjectDetectionData.from_fiftyone( + datamodule = ObjectDetectionData.fiftyone_from_datasets( train_dataset=train_dataset, val_dataset=train_dataset, test_dataset=train_dataset, diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index a20a4c06d3..abfbccf7d9 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -63,7 +63,7 @@ def test_detection_fiftyone(tmpdir, model, backbone): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - data = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) + data = ObjectDetectionData.fiftyone_from_datasets(train_dataset=train_dataset, batch_size=1) model = ObjectDetector(model=model, backbone=backbone, num_classes=data.num_classes) trainer = flash.Trainer(fast_dev_run=True) diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index edae4d42a4..9a52244866 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -252,7 +252,7 @@ def test_from_files_warning(self, tmpdir): ) @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") - def test_from_fiftyone(self, tmpdir): + def test_fiftyone_from_datasets(self, tmpdir): tmp_dir = Path(tmpdir) # create random dummy data @@ -286,7 +286,7 @@ def test_from_fiftyone(self, tmpdir): # instantiate the data module - dm = SemanticSegmentationData.from_fiftyone( + dm = SemanticSegmentationData.fiftyone_from_datasets( train_dataset=dataset, val_dataset=dataset, test_dataset=dataset, diff --git a/tests/video/test_video_classifier.py b/tests/video/test_video_classifier.py index f188e3f968..22803e7c56 100644 --- a/tests/video/test_video_classifier.py +++ b/tests/video/test_video_classifier.py @@ -13,8 +13,8 @@ # limitations under the License. import contextlib import os -from pathlib import Path import tempfile +from pathlib import Path import pytest import torch @@ -204,7 +204,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): dir_name, dataset_type=fo.types.VideoClassificationDirectoryTree, ) - datamodule = VideoClassificationData.from_fiftyone( + datamodule = VideoClassificationData.fiftyone_from_datasets( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, @@ -243,7 +243,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): ]), } - datamodule = VideoClassificationData.from_fiftyone( + datamodule = VideoClassificationData.fiftyone_from_datasets( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, From f204bb2afbdedba8d38fffd47664089dd405476d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Jun 2021 13:47:07 +0000 Subject: [PATCH 30/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flash/image/detection/serialization.py | 12 +++++------- flash/image/segmentation/data.py | 6 ++---- tests/image/detection/test_serialization.py | 3 ++- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index df53574ce5..24b6a76044 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -73,12 +73,10 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: else: label = str(int(label)) - detections.append( - Detection( - label=label, - bounding_box=box, - confidence=confidence, - ) - ) + detections.append(Detection( + label=label, + bounding_box=box, + confidence=confidence, + )) return Detections(detections=detections) diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index e5ddc80218..01cf055d61 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -43,8 +43,8 @@ if _FIFTYONE_AVAILABLE: import fiftyone as fo - from fiftyone.core.labels import Segmentation from fiftyone.core.collections import SampleCollection + from fiftyone.core.labels import Segmentation else: fo, Segmentation, SampleCollection = None, None, None @@ -171,9 +171,7 @@ def __init__(self, label_field: str = "ground_truth"): def label_cls(self): return Segmentation - def load_data(self, - data: SampleCollection, - dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: self._validate(data) self._fo_dataset_name = data.name diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py index cdaa28b3f9..9c8058d57f 100644 --- a/tests/image/detection/test_serialization.py +++ b/tests/image/detection/test_serialization.py @@ -16,7 +16,8 @@ def test_serialize_fiftyone(self): serial = FiftyOneDetectionLabels() sample = [{ - "boxes": [torch.tensor(20), torch.tensor(30), torch.tensor(40), torch.tensor(50)], + "boxes": [torch.tensor(20), torch.tensor(30), + torch.tensor(40), torch.tensor(50)], "labels": torch.tensor(0), "scores": torch.tensor(0.5), }] From d282d42c26ad3c10ced2dc8b1e62f81523f93dfc Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Mon, 7 Jun 2021 17:32:36 -0400 Subject: [PATCH 31/54] normalize detection labels --- flash/image/detection/data.py | 8 ++++++-- flash/image/detection/model.py | 7 ++++++- flash/image/detection/serialization.py | 26 ++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 7c1ca5abb0..9997da887f 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -92,7 +92,9 @@ def load_data(self, data: Tuple[str, str], dataset: Optional[Any] = None) -> Seq return data def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + img = default_loader(sample[DefaultDataKeys.INPUT]) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample @@ -157,7 +159,9 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Se return output_data def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + img = default_loader(sample[DefaultDataKeys.INPUT]) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): diff --git a/flash/image/detection/model.py b/flash/image/detection/model.py index 097bff8917..ffe5e54cb4 100644 --- a/flash/image/detection/model.py +++ b/flash/image/detection/model.py @@ -18,11 +18,13 @@ from torch.optim import Optimizer from flash.core.data.data_source import DefaultDataKeys +from flash.core.data.process import Serializer from flash.core.model import Task from flash.core.registry import FlashRegistry from flash.core.utilities.imports import _IMAGE_AVAILABLE from flash.image.backbones import OBJ_DETECTION_BACKBONES from flash.image.detection.finetuning import ObjectDetectionFineTuning +from flash.image.detection.serialization import DetectionLabels if _IMAGE_AVAILABLE: import torchvision @@ -90,6 +92,7 @@ def __init__( metrics: Union[Callable, nn.Module, Mapping, Sequence, None] = None, optimizer: Type[Optimizer] = torch.optim.AdamW, learning_rate: float = 1e-3, + serializer: Optional[Union[Serializer, Mapping[str, Serializer]]] = None, **kwargs: Any, ): @@ -112,6 +115,7 @@ def __init__( metrics=metrics, learning_rate=learning_rate, optimizer=optimizer, + serializer=serializer or DetectionLabels(), ) @staticmethod @@ -194,7 +198,8 @@ def test_step(self, batch, batch_idx): def predict_step(self, batch: Any, batch_idx: int, dataloader_idx: int = 0) -> Any: images = batch[DefaultDataKeys.INPUT] - return self.model(images) + batch[DefaultDataKeys.PREDS] = self.model(images) + return batch def configure_finetune_callback(self): return [ObjectDetectionFineTuning(train_bn=True)] diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 24b6a76044..29d4c45d20 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -15,7 +15,7 @@ from pytorch_lightning.utilities import rank_zero_warn -from flash.core.data.data_source import LabelsState +from flash.core.data.data_source import DefaultDataKeys, LabelsState from flash.core.data.process import Serializer from flash.core.utilities.imports import _FIFTYONE_AVAILABLE @@ -25,6 +25,13 @@ Detection, Detections = None, None +class DetectionLabels(Serializer): + """A :class:`.Serializer` which extracts predictions from sample dict.""" + + def serialize(self, sample: Dict[str, Any]) -> Dict[str, Any]: + return sample[DefaultDataKeys.PREDS] + + class FiftyOneDetectionLabels(Serializer): """A :class:`.Serializer` which converts model outputs to FiftyOne detection format. @@ -45,7 +52,11 @@ def __init__(self, labels: Optional[List[str]] = None, threshold: Optional[float if labels is not None: self.set_state(LabelsState(labels)) - def serialize(self, sample: List[Dict[str, Any]]) -> Detections: + def serialize(self, sample: Dict[str, Any]) -> Detections: + if DefaultDataKeys.METADATA not in sample: + raise ValueError("sample requires DefaultDataKeys.METADATA to use " + "a FiftyOneDetectionLabels serializer.") + labels = None if self._labels is not None: labels = self._labels @@ -56,16 +67,23 @@ def serialize(self, sample: List[Dict[str, Any]]) -> Detections: else: rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) + width, height = sample[DefaultDataKeys.METADATA] + detections = [] - for det in sample: + for det in sample[DefaultDataKeys.PREDS]: confidence = det["scores"].tolist() if self.threshold is not None and confidence < self.threshold: continue xmin, ymin, xmax, ymax = [c.tolist() for c in det["boxes"]] - box = [xmin, ymin, xmax - xmin, ymax - ymin] + box = [ + xmin / width, + ymin / height, + (xmax - xmin) / width, + (ymax - ymin) / height, + ] label = det["labels"].tolist() if labels is not None: From aa2de9786eb9ed16849c31961339f0e36963555a Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Mon, 7 Jun 2021 17:37:12 -0400 Subject: [PATCH 32/54] normalize detection labels --- flash/core/data/process.py | 2 +- flash/image/data.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/flash/core/data/process.py b/flash/core/data/process.py index 3002ee4275..2d0c0aadee 100644 --- a/flash/core/data/process.py +++ b/flash/core/data/process.py @@ -500,7 +500,7 @@ def _save_sample(self, sample: Any) -> None: class Serializer(Properties): - """A :class:`.Serializer` encapsulates a single ``serialize`` method which is used to convert the model ouptut into + """A :class:`.Serializer` encapsulates a single ``serialize`` method which is used to convert the model output into the desired output format when predicting.""" def __init__(self): diff --git a/flash/image/data.py b/flash/image/data.py index 295815692a..bd0df14044 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -37,26 +37,34 @@ def __init__(self): super().__init__(extensions=IMG_EXTENSIONS) def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + img = default_loader(sample[DefaultDataKeys.INPUT]) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample class ImageTensorDataSource(TensorDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = to_pil_image(sample[DefaultDataKeys.INPUT]) + img = to_pil_image(sample[DefaultDataKeys.INPUT]) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample class ImageNumpyDataSource(NumpyDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) + img = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample class ImageFiftyOneDataSource(FiftyOneDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - sample[DefaultDataKeys.INPUT] = default_loader(sample[DefaultDataKeys.INPUT]) + img = default_loader(sample[DefaultDataKeys.INPUT]) + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = img.size # WxH return sample From d56e5d3471d0d5ad1f07710cb86c866a055c2424 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 8 Jun 2021 20:30:48 -0400 Subject: [PATCH 33/54] fiftyone serializer return filepaths, paths data sources store filepaths --- flash/core/classification.py | 17 ++++-- flash/core/data/data_source.py | 9 ++- flash/core/integrations/fiftyone/utils.py | 68 ++++++++++++++++++++--- flash/image/data.py | 12 ++-- flash/image/detection/data.py | 6 +- flash/image/detection/serialization.py | 28 ++++++++-- flash/image/segmentation/data.py | 5 +- flash/image/segmentation/serialization.py | 37 ++++++++---- flash/video/classification/data.py | 3 +- flash/video/classification/model.py | 4 +- 10 files changed, 147 insertions(+), 42 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 84c1c572d8..11985198be 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union import torch import torch.nn.functional as F @@ -185,6 +185,9 @@ class FiftyOneLabels(ClassificationSerializer): threshold: A threshold to use to filter candidate labels. In the single label case, predictions below this threshold will be replaced with None store_logits: Boolean determining whether to store logits in the FiftyOne labels + return_filepath: Boolean determining whether to return a dict + containing filepath and FiftyOne labels (True) or only a + list of FiftyOne labels (False) """ def __init__( @@ -193,6 +196,7 @@ def __init__( multi_label: bool = False, threshold: Optional[float] = None, store_logits: bool = False, + return_filepath: bool = False, ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") @@ -204,14 +208,15 @@ def __init__( self._labels = labels self.threshold = threshold self.store_logits = store_logits + self.return_filepath = return_filepath if labels is not None: self.set_state(LabelsState(labels)) - def serialize(self, sample: Any) -> Union[Classification, Classifications]: + def serialize(self, sample: Any) -> Union[Classification, Classifications, Tuple[str, Classification], Tuple[str, Classifications]]: pred = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample pred = torch.tensor(pred) - metadata = sample[DefaultDataKeys.METADATA] + labels = None if self._labels is not None: @@ -285,4 +290,8 @@ def serialize(self, sample: Any) -> Union[Classification, Classifications]: logits=logits, ) - return fo.Sample(filepath=metadata.filepath, predictions=fo_predictions) + if self.return_filepath: + filepath = sample[DefaultDataKeys.FILEPATH] + return (filepath, fo_predictions) + else: + return fo_predictions diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 6ba58240f8..2db7b564d4 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -167,6 +167,7 @@ class DefaultDataKeys(LightningEnum): PREDS = "preds" TARGET = "target" METADATA = "metadata" + FILEPATH = "filepath" # TODO: Create a FlashEnum class??? def __hash__(self) -> int: @@ -454,10 +455,12 @@ def predict_load_data(self, if not isinstance(data, list): data = [data] + data = [{DefaultDataKeys.INPUT: input, DefaultDataKeys.FILEPATH: input} for input in data] + return list( filter( lambda sample: has_file_allowed_extension(sample[DefaultDataKeys.INPUT], self.extensions), - super().predict_load_data(data), + data, ) ) @@ -513,11 +516,11 @@ def to_idx(t): return [{ DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t), - DefaultDataKeys.METADATA: data[f] + DefaultDataKeys.FILEPATH: f, } for f, t in zip(filepaths, targets)] def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.METADATA: data[f]} for f in data.values("filepath")] + return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.FILEPATH: f} for f in data.values("filepath")] def _validate(self, data): label_type = data._get_label_field_type(self.label_field) diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 042433a764..804ccfb8ea 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -1,29 +1,81 @@ -from time import sleep as sleep_fn -from typing import List, Optional +from itertools import chain +from typing import List, Optional, Tuple, Union import flash +from flash.core.data.data_module import DataModule +from flash.core.data.data_source import DefaultDataKeys from flash.core.utilities.imports import _FIFTYONE_AVAILABLE if _FIFTYONE_AVAILABLE: import fiftyone as fo - from fiftyone import Sample + from fiftyone.core.labels import Label + from fiftyone.core.sample import Sample from fiftyone.core.session import Session + from fiftyone.utils.data.parsers import LabeledImageTupleSampleParser else: + fo = None + SampleCollection = None + Label = None Sample = None Session = None -def fiftyone_launch_app(samples: List[Sample], sleep: Optional[int] = 120, **kwargs) -> Optional[Session]: +def fiftyone_launch_app( + labels: Union[List[Label], List[Tuple[str, Label]]], + filepaths: Optional[List[str]] = None, + datamodule: Optional[DataModule] = None, + wait: Optional[bool] = True, + label_field: Optional[str] = "predictions", + **kwargs + ) -> Optional[Session]: + """Use the result of a FiftyOne serializer to visualize predictions in the + FiftyOne App. + + Args: + labels: Either a list of FiftyOne labels that will be applied to the + corresponding filepaths provided with through `filepath` or + `datamodule`. Or a list of tuples containing image/video + filepaths and corresponding FiftyOne labels. + filepaths: A list of filepaths to images or videos corresponding to the + provided `labels`. + datamodule: The datamodule containing the prediction dataset used to + generate `labels`. + wait: A boolean determining whether to launch the FiftyOne session and + wait until the session is closed or whether to return immediately. + label_field: The string of the label field in the FiftyOne dataset + containing predictions + """ if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, `pip install fiftyone`.") if flash._IS_TESTING: return None + + # Flatten list if batches were used + if all(isinstance(fl, list) for fl in labels): + labels = list(chain.from_iterable(labels)) + + if all(isinstance(fl, tuple) for fl in labels): + filepaths = [lab[0] for lab in labels] + labels = [lab[1] for lab in labels] + + if filepaths is None: + if datamodule is None: + raise ValueError("Either `filepaths` or `datamodule` arguments are " + "required if filepaths are not provided in `labels`.") + + else: + filepaths = [s[DefaultDataKeys.FILEPATH] for s in datamodule.predict_dataset.data] + dataset = fo.Dataset() - for sample in samples: - dataset.add_samples(sample) + if filepaths: + dataset.add_labeled_images( + list(zip(filepaths, labels)), + LabeledImageTupleSampleParser(), + label_field = label_field, + ) session = fo.launch_app(dataset, **kwargs) - if sleep: - sleep_fn(sleep) + if wait: + session.wait() return session diff --git a/flash/image/data.py b/flash/image/data.py index bd0df14044..e15901ebe9 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -39,7 +39,8 @@ def __init__(self): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample @@ -48,7 +49,8 @@ class ImageTensorDataSource(TensorDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = to_pil_image(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample @@ -57,7 +59,8 @@ class ImageNumpyDataSource(NumpyDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample @@ -66,5 +69,6 @@ class ImageFiftyOneDataSource(FiftyOneDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 9997da887f..61edd1b1fc 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -94,7 +94,8 @@ def load_data(self, data: Tuple[str, str], dataset: Optional[Any] = None) -> Seq def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample @@ -161,7 +162,8 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Se def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.size # WxH + w, h = img.size # WxH + sample[DefaultDataKeys.METADATA] = (h,w) return sample def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 29d4c45d20..1f6cc28f03 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple, Union from pytorch_lightning.utilities import rank_zero_warn @@ -39,24 +39,36 @@ class FiftyOneDetectionLabels(Serializer): labels: A list of labels, assumed to map the class index to the label for that class. If ``labels`` is not provided, will attempt to get them from the :class:`.LabelsState`. threshold: a score threshold to apply to candidate detections. + return_filepath: Boolean determining whether to return a dict + containing filepath and FiftyOne labels (True) or only a + list of FiftyOne labels (False) """ - def __init__(self, labels: Optional[List[str]] = None, threshold: Optional[float] = None): + def __init__( + self, + labels: Optional[List[str]] = None, + threshold: Optional[float] = None, + return_filepath: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") super().__init__() self._labels = labels self.threshold = threshold + self.return_filepath = return_filepath if labels is not None: self.set_state(LabelsState(labels)) - def serialize(self, sample: Dict[str, Any]) -> Detections: + def serialize(self, sample: Dict[str, Any]) -> Union[Detections, Tuple[str, Detection]]: if DefaultDataKeys.METADATA not in sample: raise ValueError("sample requires DefaultDataKeys.METADATA to use " "a FiftyOneDetectionLabels serializer.") + if self.return_filepath: + filepath = sample[DefaultDataKeys.FILEPATH] + labels = None if self._labels is not None: labels = self._labels @@ -67,7 +79,7 @@ def serialize(self, sample: Dict[str, Any]) -> Detections: else: rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) - width, height = sample[DefaultDataKeys.METADATA] + height, width = sample[DefaultDataKeys.METADATA] detections = [] @@ -96,5 +108,9 @@ def serialize(self, sample: Dict[str, Any]) -> Detections: bounding_box=box, confidence=confidence, )) - - return Detections(detections=detections) + fo_predictions = Detections(detections=detections) + if self.return_filepath: + filepath = sample[DefaultDataKeys.FILEPATH] + return (filepath, fo_predictions) + else: + return fo_predictions diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 01cf055d61..cf54503d16 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -175,7 +175,7 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Se self._validate(data) self._fo_dataset_name = data.name - return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] + return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.FILEPATH: f} for f in data.values("filepath")] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: _fo_dataset = fo.load_dataset(self._fo_dataset_name) @@ -334,7 +334,8 @@ def from_data_source( **preprocess_kwargs ) - dm.train_dataset.num_classes = num_classes + if dm.train_dataset is not None: + dm.train_dataset.num_classes = num_classes return dm @classmethod diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 0e426f404c..ca8e1f0807 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Dict, Optional, Tuple +from typing import Dict, Optional, Tuple, Union import torch @@ -88,19 +88,34 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> torch.Tensor: class FiftyOneSegmentationLabels(SegmentationLabels): - - def __init__(self, labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, visualize: bool = False): - """A :class:`.Serializer` which converts the model outputs to FiftyOne segmentation format. - - Args: - labels_map: A dictionary that map the labels ids to pixel intensities. - visualize: Wether to visualize the image labels. - """ + """A :class:`.Serializer` which converts the model outputs to FiftyOne segmentation format. + + Args: + labels_map: A dictionary that map the labels ids to pixel intensities. + visualize: Wether to visualize the image labels. + return_filepath: Boolean determining whether to return a dict + containing filepath and FiftyOne labels (True) or only a + list of FiftyOne labels (False) + """ + + def __init__( + self, + labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, + visualize: bool = False, + return_filepath: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") super().__init__(labels_map=labels_map, visualize=visualize) - def serialize(self, sample: Dict[str, torch.Tensor]) -> Segmentation: + self.return_filepath = return_filepath + + def serialize(self, sample: Dict[str, torch.Tensor]) -> Union[Segmentation, Tuple[str, Segmentation]]: labels = super().serialize(sample) - return Segmentation(mask=labels.numpy()) + fo_predictions = Segmentation(mask=labels.numpy()) + if self.return_filepath: + filepath = sample[DefaultDataKeys.FILEPATH] + return (filepath, fo_predictions) + else: + return fo_predictions diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index c4af6c13b3..0ceb5ae840 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -145,7 +145,8 @@ def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDa return ds def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - return self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT])) + sample.update(self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT]))) + return sample class VideoClassificationFiftyOneDataSource(VideoClassificationPathsDataSource): diff --git a/flash/video/classification/model.py b/flash/video/classification/model.py index 194dae99bc..4601bcaf5a 100644 --- a/flash/video/classification/model.py +++ b/flash/video/classification/model.py @@ -27,6 +27,7 @@ import flash from flash.core.classification import ClassificationTask, Labels +from flash.core.data.data_source import DefaultDataKeys from flash.core.data.process import Serializer from flash.core.registry import FlashRegistry from flash.core.utilities.imports import _PYTORCHVIDEO_AVAILABLE @@ -153,7 +154,8 @@ def forward(self, x: Any) -> Any: def predict_step(self, batch: Any, batch_idx: int, dataloader_idx: int = 0) -> Any: predictions = self(batch["video"]) - return predictions + batch[DefaultDataKeys.PREDS] = predictions + return batch def configure_finetune_callback(self) -> List[Callback]: return [VideoClassifierFinetuning()] From 55b11c6c70debb3548fe683684f97ec55199238f Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 8 Jun 2021 20:35:41 -0400 Subject: [PATCH 34/54] formatting --- flash/core/integrations/fiftyone/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 804ccfb8ea..2f4806f919 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -21,13 +21,13 @@ def fiftyone_launch_app( - labels: Union[List[Label], List[Tuple[str, Label]]], - filepaths: Optional[List[str]] = None, - datamodule: Optional[DataModule] = None, - wait: Optional[bool] = True, - label_field: Optional[str] = "predictions", - **kwargs - ) -> Optional[Session]: + labels: Union[List[Label], List[Tuple[str, Label]]], + filepaths: Optional[List[str]] = None, + datamodule: Optional[DataModule] = None, + wait: Optional[bool] = True, + label_field: Optional[str] = "predictions", + **kwargs +) -> Optional[Session]: """Use the result of a FiftyOne serializer to visualize predictions in the FiftyOne App. @@ -71,7 +71,7 @@ def fiftyone_launch_app( dataset.add_labeled_images( list(zip(filepaths, labels)), LabeledImageTupleSampleParser(), - label_field = label_field, + label_field=label_field, ) session = fo.launch_app(dataset, **kwargs) if wait: From 8e5402163304d17b121bd0001c6a5b3cfb073c5e Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Wed, 9 Jun 2021 10:35:29 -0400 Subject: [PATCH 35/54] formatting --- flash/core/classification.py | 5 ++++- flash/image/data.py | 8 ++++---- flash/image/detection/data.py | 4 ++-- flash/image/detection/serialization.py | 5 +++-- flash/image/segmentation/serialization.py | 10 +++++----- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 11985198be..3a963acce6 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -213,7 +213,10 @@ def __init__( if labels is not None: self.set_state(LabelsState(labels)) - def serialize(self, sample: Any) -> Union[Classification, Classifications, Tuple[str, Classification], Tuple[str, Classifications]]: + def serialize( + self, + sample: Any, + ) -> Union[Classification, Classifications, Tuple[str, Classification], Tuple[str, Classifications]]: pred = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample pred = torch.tensor(pred) diff --git a/flash/image/data.py b/flash/image/data.py index e15901ebe9..aaca188962 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -40,7 +40,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample @@ -50,7 +50,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = to_pil_image(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample @@ -60,7 +60,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample @@ -70,5 +70,5 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 61edd1b1fc..1884d3ae73 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -95,7 +95,7 @@ def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample @@ -163,7 +163,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = default_loader(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h,w) + sample[DefaultDataKeys.METADATA] = (h, w) return sample def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 1f6cc28f03..7f244c0728 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -28,8 +28,9 @@ class DetectionLabels(Serializer): """A :class:`.Serializer` which extracts predictions from sample dict.""" - def serialize(self, sample: Dict[str, Any]) -> Dict[str, Any]: - return sample[DefaultDataKeys.PREDS] + def serialize(self, sample: Any) -> Dict[str, Any]: + sample = sample[DefaultDataKeys.PREDS] if isinstance(sample, Dict) else sample + return sample class FiftyOneDetectionLabels(Serializer): diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index ca8e1f0807..2e12cc78a1 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -99,11 +99,11 @@ class FiftyOneSegmentationLabels(SegmentationLabels): """ def __init__( - self, - labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, - visualize: bool = False, - return_filepath: bool = False, - ): + self, + labels_map: Optional[Dict[int, Tuple[int, int, int]]] = None, + visualize: bool = False, + return_filepath: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") From 1e1022bc9b574860480f1a86348b21964424b3df Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Wed, 9 Jun 2021 11:41:24 -0400 Subject: [PATCH 36/54] remove fiftyone from dir, rename fiftyone_visualize() --- flash/core/data/data_module.py | 133 +------------------ flash/core/integrations/fiftyone/__init__.py | 2 +- flash/core/integrations/fiftyone/utils.py | 2 +- 3 files changed, 4 insertions(+), 133 deletions(-) diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 327896123a..3fb3925e64 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -1035,7 +1035,7 @@ def from_datasets( ) @classmethod - def fiftyone_from_datasets( + def from_fiftyone_datasets( cls, train_dataset: Optional[SampleCollection] = None, val_dataset: Optional[SampleCollection] = None, @@ -1091,7 +1091,7 @@ def fiftyone_from_datasets( "/path/to/dataset", dataset_type=fo.types.ImageClassificationDirectoryTree, ) - data_module = DataModule.fiftyone_from_datasets( + data_module = DataModule.from_fiftyone_datasets( train_data = train_dataset, train_transform={ "to_tensor_transform": torch.as_tensor, @@ -1118,132 +1118,3 @@ def fiftyone_from_datasets( num_workers=num_workers, **preprocess_kwargs, ) - - @classmethod - def fiftyone_from_dir( - cls, - train_dir: Optional[SampleCollection] = None, - val_dir: Optional[SampleCollection] = None, - test_dir: Optional[SampleCollection] = None, - predict_dir: Optional[SampleCollection] = None, - train_transform: Optional[Dict[str, Callable]] = None, - val_transform: Optional[Dict[str, Callable]] = None, - test_transform: Optional[Dict[str, Callable]] = None, - predict_transform: Optional[Dict[str, Callable]] = None, - data_fetcher: Optional[BaseDataFetcher] = None, - preprocess: Optional[Preprocess] = None, - val_split: Optional[float] = None, - batch_size: int = 4, - num_workers: Optional[int] = None, - dataset_type: Optional[Dataset] = None, - predict_dataset_type: Optional[Dataset] = None, - label_field: str = 'ground_truth', - tags: List[str] = None, - importer_kwargs: Dict[str, Any] = {}, - predict_importer_kwargs: Dict[str, Any] = {}, - **preprocess_kwargs: Any, - ) -> 'DataModule': - """Creates a :class:`~flash.core.data.data_module.DataModule` object - from the given FiftyOne Datasets using the - :class:`~flash.core.data.data_source.DataSource` of name - :attr:`~flash.core.data.data_source.DefaultDataSources.FIFTYONE` - from the passed or constructed :class:`~flash.core.data.process.Preprocess`. - - Args: - train_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the train data. - val_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the validation data. - test_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the test data. - predict_dataset: The ``fiftyone.core.collections.SampleCollection`` containing the predict data. - train_transform: The dictionary of transforms to use during training which maps - :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. - val_transform: The dictionary of transforms to use during validation which maps - :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. - test_transform: The dictionary of transforms to use during testing which maps - :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. - predict_transform: The dictionary of transforms to use during predicting which maps - :class:`~flash.core.data.process.Preprocess` hook names to callable transforms. - data_fetcher: The :class:`~flash.core.data.callback.BaseDataFetcher` to pass to the - :class:`~flash.core.data.data_module.DataModule`. - preprocess: The :class:`~flash.core.data.data.Preprocess` to pass to the - :class:`~flash.core.data.data_module.DataModule`. If ``None``, ``cls.preprocess_cls`` - will be constructed and used. - val_split: The ``val_split`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. - batch_size: The ``batch_size`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. - num_workers: The ``num_workers`` argument to pass to the :class:`~flash.core.data.data_module.DataModule`. - preprocess_kwargs: Additional keyword arguments to use when constructing the preprocess. Will only be used - if ``preprocess = None``. - - Returns: - The constructed data module. - - Examples:: - - train_dataset = fo.Dataset.from_dir( - "/path/to/dataset", - dataset_type=fo.types.ImageClassificationDirectoryTree, - ) - data_module = DataModule.fiftyone_from_datasets( - train_data = train_dataset, - train_transform={ - "to_tensor_transform": torch.as_tensor, - }, - ) - """ - if not _FIFTYONE_AVAILABLE: - raise ModuleNotFoundError("Please, `pip install fiftyone`.") - - if not dataset_type: - dataset_type = fo.types.ImageClassificationDirectoryTree - - if not predict_dataset_type: - predict_dataset_type = fo.types.ImageDirectory - - train_dataset = fo.Dataset.from_dir( - train_dir, - dataset_type, - label_field=label_field, - tags=tags, - **importer_kwargs, - ) - - val_dataset = fo.Dataset.from_dir( - val_dir, - dataset_type, - label_field=label_field, - tags=tags, - **importer_kwargs, - ) - - test_dataset = fo.Dataset.from_dir( - test_dir, - dataset_type, - label_field=label_field, - tags=tags, - **importer_kwargs, - ) - - predict_dataset = fo.Dataset.from_dir( - predict_dir, - predict_dataset_type, - label_field=label_field, - tags=tags, - **predict_importer_kwargs, - ) - - return cls.fiftyone_from_datasets( - train_dataset=train_dataset, - val_dataset=val_dataset, - test_dataset=test_dataset, - predict_dataset=predict_dataset, - train_transform=train_transform, - val_transform=val_transform, - test_transform=test_transform, - predict_transform=predict_transform, - data_fetcher=data_fetcher, - preprocess=preprocess, - val_split=val_split, - batch_size=batch_size, - num_workers=num_workers, - label_field=label_field, - **preprocess_kwargs, - ) diff --git a/flash/core/integrations/fiftyone/__init__.py b/flash/core/integrations/fiftyone/__init__.py index 17e27252e6..54fcd69073 100644 --- a/flash/core/integrations/fiftyone/__init__.py +++ b/flash/core/integrations/fiftyone/__init__.py @@ -1 +1 @@ -from flash.core.integrations.fiftyone.utils import fiftyone_launch_app, get_classes +from flash.core.integrations.fiftyone.utils import fiftyone_visualize, get_classes diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 2f4806f919..5029fe9ca0 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -20,7 +20,7 @@ Session = None -def fiftyone_launch_app( +def fiftyone_visualize( labels: Union[List[Label], List[Tuple[str, Label]]], filepaths: Optional[List[str]] = None, datamodule: Optional[DataModule] = None, From 7c44cb58d6264e3a84b8e2c09ed8d5d6d245298b Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Fri, 11 Jun 2021 17:34:33 -0400 Subject: [PATCH 37/54] update metadata to contain dictionaries --- flash/core/classification.py | 2 +- flash/core/data/data_module.py | 9 ++-- flash/core/data/data_source.py | 6 +-- flash/core/integrations/fiftyone/utils.py | 2 +- flash/image/data.py | 20 +++++--- flash/image/detection/data.py | 17 +++++-- flash/image/detection/serialization.py | 17 +++---- flash/image/segmentation/data.py | 50 ++++++++++++------- flash/image/segmentation/model.py | 2 +- flash/image/segmentation/serialization.py | 2 +- flash/video/classification/data.py | 8 +-- flash_examples/predict/image_embedder.py | 2 + .../fiftyone/image_classification.py | 16 +++--- tests/core/test_classification.py | 13 ++--- tests/image/classification/test_data.py | 4 +- .../test_data_model_integration.py | 2 +- tests/image/detection/test_data.py | 6 +-- .../detection/test_data_model_integration.py | 2 +- tests/image/detection/test_serialization.py | 23 ++++++--- tests/image/segmentation/test_data.py | 4 +- tests/video/classification/test_model.py | 4 +- 21 files changed, 122 insertions(+), 89 deletions(-) diff --git a/flash/core/classification.py b/flash/core/classification.py index 7efe2fbb4b..350a6a0118 100644 --- a/flash/core/classification.py +++ b/flash/core/classification.py @@ -294,7 +294,7 @@ def serialize( ) if self.return_filepath: - filepath = sample[DefaultDataKeys.FILEPATH] + filepath = sample[DefaultDataKeys.METADATA]["filepath"] return {"filepath": filepath, "predictions": fo_predictions} else: return fo_predictions diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 20f3afaf20..72961fbca8 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -36,10 +36,8 @@ if _FIFTYONE_AVAILABLE: import fiftyone as fo from fiftyone.core.collections import SampleCollection - from fiftyone.types import Dataset else: SampleCollection = None - Dataset = None class DataModule(pl.LightningDataModule): @@ -324,8 +322,8 @@ def _predict_dataloader(self) -> DataLoader: @property def num_classes(self) -> Optional[int]: return ( - getattr(self.train_dataset, "num_classes", None) or getattr(self.val_dataset, "num_classes", None) - or getattr(self.test_dataset, "num_classes", None) + getattr(self.train_dataset, "num_classes", None) or getattr(self.val_dataset, "num_classes", None) or + getattr(self.test_dataset, "num_classes", None) ) @property @@ -345,7 +343,8 @@ def data_pipeline(self) -> DataPipeline: return DataPipeline(self.data_source, self.preprocess, self.postprocess) def available_data_sources(self) -> Sequence[str]: - """Get the list of available data source names for use with this :class:`~flash.core.data.data_module.DataModule`. + """Get the list of available data source names for use with this + :class:`~flash.core.data.data_module.DataModule`. Returns: The list of data source names. diff --git a/flash/core/data/data_source.py b/flash/core/data/data_source.py index 2db7b564d4..c29a86bfd3 100644 --- a/flash/core/data/data_source.py +++ b/flash/core/data/data_source.py @@ -167,7 +167,6 @@ class DefaultDataKeys(LightningEnum): PREDS = "preds" TARGET = "target" METADATA = "metadata" - FILEPATH = "filepath" # TODO: Create a FlashEnum class??? def __hash__(self) -> int: @@ -455,7 +454,7 @@ def predict_load_data(self, if not isinstance(data, list): data = [data] - data = [{DefaultDataKeys.INPUT: input, DefaultDataKeys.FILEPATH: input} for input in data] + data = [{DefaultDataKeys.INPUT: input} for input in data] return list( filter( @@ -516,11 +515,10 @@ def to_idx(t): return [{ DefaultDataKeys.INPUT: f, DefaultDataKeys.TARGET: to_idx(t), - DefaultDataKeys.FILEPATH: f, } for f, t in zip(filepaths, targets)] def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.FILEPATH: f} for f in data.values("filepath")] + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def _validate(self, data): label_type = data._get_label_field_type(self.label_field) diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 25410301b6..9ed01f468b 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -1,5 +1,5 @@ from itertools import chain -from typing import List, Optional, Tuple, Union +from typing import Dict, List, Optional, Union import flash from flash.core.data.data_module import DataModule diff --git a/flash/image/data.py b/flash/image/data.py index aaca188962..69fd25e657 100644 --- a/flash/image/data.py +++ b/flash/image/data.py @@ -37,10 +37,14 @@ def __init__(self): super().__init__(extensions=IMG_EXTENSIONS) def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - img = default_loader(sample[DefaultDataKeys.INPUT]) + img_path = sample[DefaultDataKeys.INPUT] + img = default_loader(img_path) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": (h, w), + } return sample @@ -50,7 +54,7 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = to_pil_image(sample[DefaultDataKeys.INPUT]) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = {"size": (h, w)} return sample @@ -60,15 +64,19 @@ def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> img = to_pil_image(torch.from_numpy(sample[DefaultDataKeys.INPUT])) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = {"size": (h, w)} return sample class ImageFiftyOneDataSource(FiftyOneDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - img = default_loader(sample[DefaultDataKeys.INPUT]) + img_path = sample[DefaultDataKeys.INPUT] + img = default_loader(img_path) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": (h, w), + } return sample diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 1884d3ae73..f7f3416bf9 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -92,10 +92,15 @@ def load_data(self, data: Tuple[str, str], dataset: Optional[Any] = None) -> Seq return data def load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - img = default_loader(sample[DefaultDataKeys.INPUT]) + filepath = sample[DefaultDataKeys.INPUT] + img = default_loader(filepath) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = { + "filepath": filepath, + "size": (h, w), + } + return sample return sample @@ -160,10 +165,14 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Se return output_data def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: - img = default_loader(sample[DefaultDataKeys.INPUT]) + filepath = sample[DefaultDataKeys.INPUT] + img = default_loader(filepath) sample[DefaultDataKeys.INPUT] = img w, h = img.size # WxH - sample[DefaultDataKeys.METADATA] = (h, w) + sample[DefaultDataKeys.METADATA] = { + "filepath": filepath, + "size": (h, w), + } return sample def _reformat_bbox(self, xmin, ymin, box_w, box_h, img_w, img_h): diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 384331b40e..73d28d8b96 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -46,11 +46,11 @@ class FiftyOneDetectionLabels(Serializer): """ def __init__( - self, - labels: Optional[List[str]] = None, - threshold: Optional[float] = None, - return_filepath: bool = False, - ): + self, + labels: Optional[List[str]] = None, + threshold: Optional[float] = None, + return_filepath: bool = False, + ): if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") @@ -67,9 +67,6 @@ def serialize(self, sample: Dict[str, Any]) -> Union[Detections, Dict[str, Any]] raise ValueError("sample requires DefaultDataKeys.METADATA to use " "a FiftyOneDetectionLabels serializer.") - if self.return_filepath: - filepath = sample[DefaultDataKeys.FILEPATH] - labels = None if self._labels is not None: labels = self._labels @@ -80,7 +77,7 @@ def serialize(self, sample: Dict[str, Any]) -> Union[Detections, Dict[str, Any]] else: rank_zero_warn("No LabelsState was found, int targets will be used as label strings", UserWarning) - height, width = sample[DefaultDataKeys.METADATA] + height, width = sample[DefaultDataKeys.METADATA]["size"] detections = [] @@ -111,7 +108,7 @@ def serialize(self, sample: Dict[str, Any]) -> Union[Detections, Dict[str, Any]] )) fo_predictions = Detections(detections=detections) if self.return_filepath: - filepath = sample[DefaultDataKeys.FILEPATH] + filepath = sample[DefaultDataKeys.METADATA]["filepath"] return {"filepath": filepath, "predictions": fo_predictions} else: return fo_predictions diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index cf54503d16..897a83bd64 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -71,7 +71,7 @@ class SemanticSegmentationNumpyDataSource(NumpyDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = torch.from_numpy(sample[DefaultDataKeys.INPUT]).float() sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.shape + sample[DefaultDataKeys.METADATA] = {"size": img.shape} return sample @@ -80,7 +80,7 @@ class SemanticSegmentationTensorDataSource(TensorDataSource): def load_sample(self, sample: Dict[str, Any], dataset: Optional[Any] = None) -> Dict[str, Any]: img = sample[DefaultDataKeys.INPUT].float() sample[DefaultDataKeys.INPUT] = img - sample[DefaultDataKeys.METADATA] = img.shape + sample[DefaultDataKeys.METADATA] = {"size": img.shape} return sample @@ -144,18 +144,24 @@ def load_sample(self, sample: Mapping[str, Any]) -> Mapping[str, Union[torch.Ten img_labels: torch.Tensor = torchvision.io.read_image(img_labels_path) # CxHxW img_labels = img_labels[0] # HxW - return { - DefaultDataKeys.INPUT: img.float(), - DefaultDataKeys.TARGET: img_labels.float(), - DefaultDataKeys.METADATA: img.shape, + sample[DefaultDataKeys.INPUT] = img.float() + sample[DefaultDataKeys.TARGET] = img_labels.float() + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": img.shape, } + return sample def predict_load_sample(self, sample: Mapping[str, Any]) -> Mapping[str, Any]: - img = torchvision.io.read_image(sample[DefaultDataKeys.INPUT]).float() - return { - DefaultDataKeys.INPUT: img, - DefaultDataKeys.METADATA: img.shape, + img_path = sample[DefaultDataKeys.INPUT] + img = torchvision.io.read_image(img_path).float() + + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": img.shape, } + return sample class SemanticSegmentationFiftyOneDataSource(FiftyOneDataSource): @@ -175,7 +181,7 @@ def load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Se self._validate(data) self._fo_dataset_name = data.name - return [{DefaultDataKeys.INPUT: f, DefaultDataKeys.FILEPATH: f} for f in data.values("filepath")] + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Tensor, torch.Size]]: _fo_dataset = fo.load_dataset(self._fo_dataset_name) @@ -186,18 +192,24 @@ def load_sample(self, sample: Mapping[str, str]) -> Mapping[str, Union[torch.Ten img: torch.Tensor = torchvision.io.read_image(img_path) # CxHxW img_labels: torch.Tensor = torch.from_numpy(fo_sample[self.label_field].mask) # HxW - return { - DefaultDataKeys.INPUT: img.float(), - DefaultDataKeys.TARGET: img_labels.float(), - DefaultDataKeys.METADATA: img.shape, + sample[DefaultDataKeys.INPUT] = img.float() + sample[DefaultDataKeys.TARGET] = img_labels.float() + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": img.shape, } + return sample def predict_load_sample(self, sample: Mapping[str, Any]) -> Mapping[str, Any]: - img = torchvision.io.read_image(sample[DefaultDataKeys.INPUT]).float() - return { - DefaultDataKeys.INPUT: img, - DefaultDataKeys.METADATA: img.shape, + img_path = sample[DefaultDataKeys.INPUT] + img = torchvision.io.read_image(img_path).float() + + sample[DefaultDataKeys.INPUT] = img + sample[DefaultDataKeys.METADATA] = { + "filepath": img_path, + "size": img.shape, } + return sample class SemanticSegmentationPreprocess(Preprocess): diff --git a/flash/image/segmentation/model.py b/flash/image/segmentation/model.py index b24d4e9476..4a8ecc44fe 100644 --- a/flash/image/segmentation/model.py +++ b/flash/image/segmentation/model.py @@ -33,7 +33,7 @@ class SemanticSegmentationPostprocess(Postprocess): def per_sample_transform(self, sample: Any) -> Any: - resize = K.geometry.Resize(sample[DefaultDataKeys.METADATA][-2:], interpolation='bilinear') + resize = K.geometry.Resize(sample[DefaultDataKeys.METADATA]["size"][-2:], interpolation='bilinear') sample[DefaultDataKeys.PREDS] = resize(torch.stack(sample[DefaultDataKeys.PREDS])) sample[DefaultDataKeys.INPUT] = resize(torch.stack(sample[DefaultDataKeys.INPUT])) return super().per_sample_transform(sample) diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index f2786c53e5..eca9d01b57 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -115,7 +115,7 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> Union[Segmentation, Dict labels = super().serialize(sample) fo_predictions = Segmentation(mask=labels.numpy()) if self.return_filepath: - filepath = sample[DefaultDataKeys.FILEPATH] + filepath = sample[DefaultDataKeys.METADATA]["filepath"] return {"filepath": filepath, "predictions": fo_predictions} else: return fo_predictions diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 0ceb5ae840..d64674e167 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -145,7 +145,9 @@ def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDa return ds def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - sample.update(self._encoded_video_to_dict(EncodedVideo.from_path(sample[DefaultDataKeys.INPUT]))) + video_path = sample[DefaultDataKeys.INPUT] + sample.update(self._encoded_video_to_dict(EncodedVideo.from_path(video_path))) + sample[DefaultDataKeys.METADATA] = {"filepath": video_path} return sample @@ -178,9 +180,9 @@ def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDa label_to_class_mapping = dict(enumerate(classes)) class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} - filepaths, labels = data.values("filepath"), data.values(self.label_field + ".label") + filepaths, labels = data.values(["filepath", self.label_field + ".label"]) targets = [class_to_label_mapping[lab] for lab in labels] - labeled_video_paths = LabeledVideoPaths(list(zip(data.values("filepath"), targets))) + labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) ds: EncodedVideoDataset = EncodedVideoDataset( labeled_video_paths, diff --git a/flash_examples/predict/image_embedder.py b/flash_examples/predict/image_embedder.py index d763766c04..1c99853dcf 100644 --- a/flash_examples/predict/image_embedder.py +++ b/flash_examples/predict/image_embedder.py @@ -1,3 +1,5 @@ +import pytorch_lightning as pl +pl.seed_everything(42) # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples_integrations/fiftyone/image_classification.py index fdc8682e87..69d38b1dba 100644 --- a/flash_examples_integrations/fiftyone/image_classification.py +++ b/flash_examples_integrations/fiftyone/image_classification.py @@ -2,18 +2,18 @@ from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data from flash.core.finetuning import FreezeUnfreeze -from flash.core.integrations.fiftyone import fiftyone_launch_app +from flash.core.integrations.fiftyone import fiftyone_visualize from flash.image import ImageClassificationData, ImageClassifier # 1 Download data download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") # 2 Load data using fiftyone -datamodule = ImageClassificationData.fiftyone_from_dir( - train_dir="data/hymenoptera_data/train/", - val_dir="data/hymenoptera_data/val/", - test_dir="data/hymenoptera_data/test/", - predict_dir="data/hymenoptera_data/predict/", +datamodule = ImageClassificationData.from_folders( + train_folder="data/hymenoptera_data/train/", + val_folder="data/hymenoptera_data/val/", + test_folder="data/hymenoptera_data/test/", + predict_folder="data/hymenoptera_data/predict/", ) # 3 Fine tune a model @@ -24,8 +24,8 @@ # 4 Predict from checkpoint model = ImageClassifier.load_from_checkpoint("https://flash-weights.s3.amazonaws.com/image_classification_model.pt") -model.serializer = FiftyOneLabels() +model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) # 5. Visualize predictions in FiftyOne for 2 minutes. -session = fiftyone_launch_app(predictions) +session = fiftyone_visualize(predictions) diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index 7a6f05fcb2..be42e661e9 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -45,22 +45,19 @@ def test_classification_serializers_multi_label(): @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") def test_classification_serializers_fiftyone(): - class MockSample(object): - filepath = "something" - logits = torch.tensor([-0.1, 0.2, 0.3]) - example_output = {DefaultDataKeys.PREDS: logits, DefaultDataKeys.METADATA: MockSample()} # 3 classes + example_output = {DefaultDataKeys.PREDS: logits, DefaultDataKeys.METADATA: {"filepath": "something"}} # 3 classes labels = ['class_1', 'class_2', 'class_3'] - predictions = FiftyOneLabels(store_logits=True).serialize(example_output).predictions + predictions = FiftyOneLabels(store_logits=True).serialize(example_output) assert torch.allclose(torch.tensor(predictions.logits), logits) assert torch.allclose(torch.tensor(predictions.confidence), torch.softmax(logits, -1)[-1]) assert predictions.label == '2' - predictions = FiftyOneLabels(labels, store_logits=True).serialize(example_output).predictions + predictions = FiftyOneLabels(labels, store_logits=True).serialize(example_output) assert predictions.label == 'class_3' - predictions = FiftyOneLabels(store_logits=True, multi_label=True).serialize(example_output).predictions + predictions = FiftyOneLabels(store_logits=True, multi_label=True).serialize(example_output) assert torch.allclose(torch.tensor(predictions.logits), logits) assert [c.label for c in predictions.classifications] == ['1', '2'] - predictions = FiftyOneLabels(labels, multi_label=True).serialize(example_output).predictions + predictions = FiftyOneLabels(labels, multi_label=True).serialize(example_output) assert [c.label for c in predictions.classifications] == ['class_2', 'class_3'] diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index f87a4bc398..c7d46cfc70 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -387,7 +387,7 @@ def test_from_data(data, from_function): @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") -def test_fiftyone_from_datasets(tmpdir): +def test_from_fiftyone_datasets(tmpdir): tmpdir = Path(tmpdir) (tmpdir / "a").mkdir() @@ -408,7 +408,7 @@ def test_fiftyone_from_datasets(tmpdir): s1.save() s2.save() - img_data = ImageClassificationData.fiftyone_from_datasets( + img_data = ImageClassificationData.from_fiftyone_datasets( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/classification/test_data_model_integration.py b/tests/image/classification/test_data_model_integration.py index 68378b8fe1..c10b1f72c0 100644 --- a/tests/image/classification/test_data_model_integration.py +++ b/tests/image/classification/test_data_model_integration.py @@ -84,7 +84,7 @@ def test_classification_fiftyone(tmpdir): s1.save() s2.save() - data = ImageClassificationData.fiftyone_from_datasets( + data = ImageClassificationData.from_fiftyone_datasets( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index 63310555e2..58ef251b18 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -168,11 +168,11 @@ def test_image_detector_data_from_coco(tmpdir): @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") -def test_image_detector_data_fiftyone_from_datasets(tmpdir): +def test_image_detector_data_from_fiftyone_datasets(tmpdir): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - datamodule = ObjectDetectionData.fiftyone_from_datasets(train_dataset=train_dataset, batch_size=1) + datamodule = ObjectDetectionData.from_fiftyone_datasets(train_dataset=train_dataset, batch_size=1) data = next(iter(datamodule.train_dataloader())) imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] @@ -185,7 +185,7 @@ def test_image_detector_data_fiftyone_from_datasets(tmpdir): assert datamodule.val_dataloader() is None assert datamodule.test_dataloader() is None - datamodule = ObjectDetectionData.fiftyone_from_datasets( + datamodule = ObjectDetectionData.from_fiftyone_datasets( train_dataset=train_dataset, val_dataset=train_dataset, test_dataset=train_dataset, diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index abfbccf7d9..5697d09019 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -63,7 +63,7 @@ def test_detection_fiftyone(tmpdir, model, backbone): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - data = ObjectDetectionData.fiftyone_from_datasets(train_dataset=train_dataset, batch_size=1) + data = ObjectDetectionData.from_fiftyone_datasets(train_dataset=train_dataset, batch_size=1) model = ObjectDetector(model=model, backbone=backbone, num_classes=data.num_classes) trainer = flash.Trainer(fast_dev_run=True) diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py index 9c8058d57f..fc600bef6f 100644 --- a/tests/image/detection/test_serialization.py +++ b/tests/image/detection/test_serialization.py @@ -1,6 +1,7 @@ import pytest import torch +from flash.core.data.data_source import DefaultDataKeys from flash.core.utilities.imports import _FIFTYONE_AVAILABLE from flash.image.detection.serialization import FiftyOneDetectionLabels @@ -15,13 +16,21 @@ def test_smoke(self): def test_serialize_fiftyone(self): serial = FiftyOneDetectionLabels() - sample = [{ - "boxes": [torch.tensor(20), torch.tensor(30), - torch.tensor(40), torch.tensor(50)], - "labels": torch.tensor(0), - "scores": torch.tensor(0.5), - }] + sample = { + DefaultDataKeys.PREDS: [ + { + "boxes": [torch.tensor(20), torch.tensor(30), + torch.tensor(40), torch.tensor(50)], + "labels": torch.tensor(0), + "scores": torch.tensor(0.5), + }, + ], + DefaultDataKeys.METADATA: { + "filepath": "something", + "size": (100, 100), + }, + } detections = serial.serialize(sample) assert len(detections.detections) == 1 - assert detections.detections[0].bounding_box == [20, 30, 20, 20] + assert detections.detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index 9a52244866..18e81a9d84 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -252,7 +252,7 @@ def test_from_files_warning(self, tmpdir): ) @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") - def test_fiftyone_from_datasets(self, tmpdir): + def test_from_fiftyone_datasets(self, tmpdir): tmp_dir = Path(tmpdir) # create random dummy data @@ -286,7 +286,7 @@ def test_fiftyone_from_datasets(self, tmpdir): # instantiate the data module - dm = SemanticSegmentationData.fiftyone_from_datasets( + dm = SemanticSegmentationData.from_fiftyone_datasets( train_dataset=dataset, val_dataset=dataset, test_dataset=dataset, diff --git a/tests/video/classification/test_model.py b/tests/video/classification/test_model.py index f245fd04c5..88ffd08fe2 100644 --- a/tests/video/classification/test_model.py +++ b/tests/video/classification/test_model.py @@ -204,7 +204,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): dir_name, dataset_type=fo.types.VideoClassificationDirectoryTree, ) - datamodule = VideoClassificationData.fiftyone_from_datasets( + datamodule = VideoClassificationData.from_fiftyone_datasets( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, @@ -243,7 +243,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): ]), } - datamodule = VideoClassificationData.fiftyone_from_datasets( + datamodule = VideoClassificationData.from_fiftyone_datasets( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, From 080d81a49c3de7632e19f3dced05f597e96ec9b5 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Fri, 11 Jun 2021 19:33:35 -0400 Subject: [PATCH 38/54] add fiftyone docs --- docs/source/conf.py | 1 + docs/source/index.rst | 7 + docs/source/integrations/fiftyone.rst | 192 ++++++++++++++++++ .../fiftyone/image_classification.py | 40 +++- 4 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 docs/source/integrations/fiftyone.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 218dd4564d..b9ad9e923f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -92,6 +92,7 @@ def _load_py_module(fname, pkg="flash"): "PIL": ("https://pillow.readthedocs.io/en/stable/", None), "pytorchvideo": ("https://pytorchvideo.readthedocs.io/en/latest/", None), "pytorch_lightning": ("https://pytorch-lightning.readthedocs.io/en/stable/", None), + "fiftyone": ("https://voxel51.com/docs/fiftyone/", None), } # -- Options for HTML output ------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 8fb3169d28..17161b93a6 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -52,6 +52,13 @@ Lightning Flash general/jit +.. toctree:: + :maxdepth: 1 + :caption: Integrations + + integrations/fiftyone + + .. toctree:: :maxdepth: 1 :caption: Contributing a Task diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst new file mode 100644 index 0000000000..380892de62 --- /dev/null +++ b/docs/source/integrations/fiftyone.rst @@ -0,0 +1,192 @@ +######## +FiftyOne +######## + +We have collaborated with the team at +`Voxel51 `_ to integrate their tool, +`FiftyOne `_, into Flash. + +FiftyOne is an open-source tool for building high-quality +datasets and computer vision models. The FiftyOne API and App enable you to +visualize datasets and interpret models faster and more effectively. + +This integration allows you to view predictions generated by Flash tasks in the +:ref:`FiftyOne App `, as well as easily incorporate +:ref:`FiftyOne Datasets ` into your Flash tasks. All image and video tasks +are supported! + +.. raw:: html + +
+ +
+ +************ +Installation +************ + +In order to utilize this integration with FiftyOne, you will need to +:ref:`install the tool`: + +.. code:: shell + + pip install fiftyone + + +*********************** +Visualizing predictions +*********************** + +This integration adds the ability to visualize and explore the predictions +generated by your tasks and models by adding the +:func:`~flash.core.integrations.fiftyone.fiftyone_visualize` +function that works with the following serializers: + +* :class:`FiftyOneLabels(return_filepath=True)` +* :class:`FiftyOneSegmentationLabels(return_filepath=True)` +* :class:`FiftyOneDetectionLabels(return_filepath=True)` + +The :func:`~flash.core.integrations.fiftyone.fiftyone_visualize` function accepts a list of +dictionaries containing :ref:`FiftyOne Label` objects +and filepaths which is the output of the FiftyOne serializers when the flag +``return_filepath=True``. + +.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification.py + :language: python + :lines: 14- + + +************************* +Loading FiftyOne datasets +************************* + +The above workflow is great for visualizing model predictions. However, if you +store your data in a FiftyOne Dataset initially, then you can also visualize +ground truth annotations allowing you to perform more complex analysis with +:ref:`views ` into your data and +:ref:`evaluation ` of your model results. + +The +:meth:`~flash.core.data_module.DataModule.from_fiftyone_datasets` +method allows you to load your FiftyOne Datasets directly into a +:class:`~flash.core.data_module.DataModule` to be used for training, +testing, or inference. + +.. code:: python + + import fiftyone as fo + + import flash + from flash.core.classification import FiftyOneLabels, Labels, Probabilities + from flash.core.data.utils import download_data + from flash.core.finetuning import FreezeUnfreeze + from flash.core.integrations.fiftyone import fiftyone_visualize + from flash.image import ImageClassificationData, ImageClassifier + + # 1 Download data + download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") + + # 2 Load data into FiftyOne + train_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/train/", + dataset_type=fo.types.ImageClassificationDirectoryTree, + ) + val_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/val/", + dataset_type=fo.types.ImageClassificationDirectoryTree, + ) + test_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/test/", + dataset_type=fo.types.ImageClassificationDirectoryTree, + ) + + # 3 Load FiftyOne datasets + datamodule = ImageClassificationData.from_fiftyone_datasets( + train_dataset=train_dataset, + val_dataset=val_dataset, + test_dataset=test_dataset, + ) + + # 4 Fine tune a model + model = ImageClassifier( + backbone="resnet18", + num_classes=datamodule.num_classes, + serializer=Labels(), + ) + trainer = flash.Trainer( + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + ) + trainer.finetune( + model, + datamodule=datamodule, + strategy=FreezeUnfreeze(unfreeze_epoch=1), + ) + trainer.save_checkpoint("image_classification_model.pt") + + # 5 Predict from checkpoint on data with ground truth + model = ImageClassifier.load_from_checkpoint( + "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" + ) + model.serializer = FiftyOneLabels(return_filepath=False) + datamodule = ImageClassificationData.from_fiftyone_datasets( + predict_dataset=test_dataset, + ) + predictions = trainer.predict(model, datamodule=datamodule) + + # 6 Add predictions to dataset + test_dataset.set_values("predictions", predictions) + + # 7 Visualize labels in the App + session = fo.launch_app(test_dataset) + + # 7b Block until the App is closed + session.wait() + + # 8 Evaluate your model + results = test_dataset.evaluate_classifications( + "predictions", + gt_field="ground_truth", + eval_key="eval", + ) + results.print_report() + plot = results.plot_confusion_matrix() + plot.show() + + + +------ + +************* +API reference +************* + +.. _fiftyone_labels: + +FiftyOneLabels +-------------- + +.. autoclass:: flash.core.classification.FiftyOneLabels + :members: + :exclude-members: forward + +.. _fiftyone_segmentation_labels: + +FiftyOneSegmentationLabels +-------------------------- + +.. autoclass:: flash.image.segmentation.serialization.FiftyOneSegmentationLabels + :members: + :exclude-members: forward + +.. _fiftyone_detection_labels: + +FiftyOneDetectionLabels +----------------------- + +.. autoclass:: flash.image.detection.serialization.FiftyOneDetectionLabels + :members: + :exclude-members: forward diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples_integrations/fiftyone/image_classification.py index 69d38b1dba..cfe3435d49 100644 --- a/flash_examples_integrations/fiftyone/image_classification.py +++ b/flash_examples_integrations/fiftyone/image_classification.py @@ -1,3 +1,16 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import flash from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data @@ -8,7 +21,7 @@ # 1 Download data download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") -# 2 Load data using fiftyone +# 2 Load data datamodule = ImageClassificationData.from_folders( train_folder="data/hymenoptera_data/train/", val_folder="data/hymenoptera_data/val/", @@ -17,15 +30,30 @@ ) # 3 Fine tune a model -model = ImageClassifier(backbone="resnet18", num_classes=datamodule.num_classes, serializer=Labels()) -trainer = flash.Trainer(max_epochs=1, limit_train_batches=1, limit_val_batches=1) -trainer.finetune(model, datamodule=datamodule, strategy=FreezeUnfreeze(unfreeze_epoch=1)) +model = ImageClassifier( + backbone="resnet18", + num_classes=datamodule.num_classes, + serializer=Labels(), +) +trainer = flash.Trainer( + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, +) +trainer.finetune( + model, + datamodule=datamodule, + strategy=FreezeUnfreeze(unfreeze_epoch=1), +) trainer.save_checkpoint("image_classification_model.pt") # 4 Predict from checkpoint -model = ImageClassifier.load_from_checkpoint("https://flash-weights.s3.amazonaws.com/image_classification_model.pt") +model = ImageClassifier.load_from_checkpoint( + "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" +) model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) -# 5. Visualize predictions in FiftyOne for 2 minutes. +# 5. Visualize predictions in FiftyOne +# Note: this blocks until the FiftyOne App is closed session = fiftyone_visualize(predictions) From f34ab3bd08b0a11ce6c2d61327ff6e8c47d89d4f Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Sat, 12 Jun 2021 10:59:24 -0400 Subject: [PATCH 39/54] update fiftyone examples --- docs/source/integrations/fiftyone.rst | 86 +------------ flash/video/classification/data.py | 5 +- .../fiftyone/image_classification.py | 5 + .../image_classification_fiftyone_datasets.py | 97 +++++++++++++++ test.py | 115 ++++++++++++++++++ 5 files changed, 224 insertions(+), 84 deletions(-) create mode 100644 flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py create mode 100644 test.py diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst index 380892de62..5747274643 100644 --- a/docs/source/integrations/fiftyone.rst +++ b/docs/source/integrations/fiftyone.rst @@ -74,89 +74,9 @@ method allows you to load your FiftyOne Datasets directly into a :class:`~flash.core.data_module.DataModule` to be used for training, testing, or inference. -.. code:: python - - import fiftyone as fo - - import flash - from flash.core.classification import FiftyOneLabels, Labels, Probabilities - from flash.core.data.utils import download_data - from flash.core.finetuning import FreezeUnfreeze - from flash.core.integrations.fiftyone import fiftyone_visualize - from flash.image import ImageClassificationData, ImageClassifier - - # 1 Download data - download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") - - # 2 Load data into FiftyOne - train_dataset = fo.Dataset.from_dir( - dataset_dir="data/hymenoptera_data/train/", - dataset_type=fo.types.ImageClassificationDirectoryTree, - ) - val_dataset = fo.Dataset.from_dir( - dataset_dir="data/hymenoptera_data/val/", - dataset_type=fo.types.ImageClassificationDirectoryTree, - ) - test_dataset = fo.Dataset.from_dir( - dataset_dir="data/hymenoptera_data/test/", - dataset_type=fo.types.ImageClassificationDirectoryTree, - ) - - # 3 Load FiftyOne datasets - datamodule = ImageClassificationData.from_fiftyone_datasets( - train_dataset=train_dataset, - val_dataset=val_dataset, - test_dataset=test_dataset, - ) - - # 4 Fine tune a model - model = ImageClassifier( - backbone="resnet18", - num_classes=datamodule.num_classes, - serializer=Labels(), - ) - trainer = flash.Trainer( - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - ) - trainer.finetune( - model, - datamodule=datamodule, - strategy=FreezeUnfreeze(unfreeze_epoch=1), - ) - trainer.save_checkpoint("image_classification_model.pt") - - # 5 Predict from checkpoint on data with ground truth - model = ImageClassifier.load_from_checkpoint( - "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" - ) - model.serializer = FiftyOneLabels(return_filepath=False) - datamodule = ImageClassificationData.from_fiftyone_datasets( - predict_dataset=test_dataset, - ) - predictions = trainer.predict(model, datamodule=datamodule) - - # 6 Add predictions to dataset - test_dataset.set_values("predictions", predictions) - - # 7 Visualize labels in the App - session = fo.launch_app(test_dataset) - - # 7b Block until the App is closed - session.wait() - - # 8 Evaluate your model - results = test_dataset.evaluate_classifications( - "predictions", - gt_field="ground_truth", - eval_key="eval", - ) - results.print_report() - plot = results.plot_confusion_matrix() - plot.show() - - +.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py + :language: python + :lines: 14- ------ diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index d64674e167..78711e6cd2 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import pathlib -from typing import Any, Callable, Dict, List, Optional, Type, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Type, Union import numpy as np import torch @@ -193,6 +193,9 @@ def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDa ) return ds + def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: + return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] + class VideoClassificationPreprocess(Preprocess): diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples_integrations/fiftyone/image_classification.py index cfe3435d49..fdcdcea87d 100644 --- a/flash_examples_integrations/fiftyone/image_classification.py +++ b/flash_examples_integrations/fiftyone/image_classification.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import itertools + import flash from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data @@ -54,6 +56,9 @@ model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) +# 4b Flatten batched predictions +predictions = list(itertools.chain.from_iterable(predictions)) + # 5. Visualize predictions in FiftyOne # Note: this blocks until the FiftyOne App is closed session = fiftyone_visualize(predictions) diff --git a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py new file mode 100644 index 0000000000..40f5a61741 --- /dev/null +++ b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -0,0 +1,97 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import itertools + +import flash +from flash.core.classification import FiftyOneLabels, Labels, Probabilities +from flash.core.data.utils import download_data +from flash.core.finetuning import FreezeUnfreeze +from flash.core.integrations.fiftyone import fiftyone_visualize +from flash.image import ImageClassificationData, ImageClassifier + +import fiftyone as fo + +# 1 Download data +download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") + +# 2 Load data into FiftyOne +train_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/train/", + dataset_type=fo.types.ImageClassificationDirectoryTree, +) +val_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/val/", + dataset_type=fo.types.ImageClassificationDirectoryTree, +) +test_dataset = fo.Dataset.from_dir( + dataset_dir="data/hymenoptera_data/test/", + dataset_type=fo.types.ImageClassificationDirectoryTree, +) + +# 3 Load FiftyOne datasets +datamodule = ImageClassificationData.from_fiftyone_datasets( + train_dataset=train_dataset, + val_dataset=val_dataset, + test_dataset=test_dataset, +) + +# 4 Fine tune a model +model = ImageClassifier( + backbone="resnet18", + num_classes=datamodule.num_classes, + serializer=Labels(), +) +trainer = flash.Trainer( + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, +) +trainer.finetune( + model, + datamodule=datamodule, + strategy=FreezeUnfreeze(unfreeze_epoch=1), +) +trainer.save_checkpoint("image_classification_model.pt") + +# 5 Predict from checkpoint on data with ground truth +model = ImageClassifier.load_from_checkpoint( + "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" +) +model.serializer = FiftyOneLabels(return_filepath=False) +datamodule = ImageClassificationData.from_fiftyone_datasets( + predict_dataset=test_dataset, +) +predictions = trainer.predict(model, datamodule=datamodule) + +# 5b Flatten batched predictions +predictions = list(itertools.chain.from_iterable(predictions)) + +# 6 Add predictions to dataset +test_dataset.set_values("predictions", predictions) + +# 7 Visualize labels in the App +session = fo.launch_app(test_dataset) + +# 7b Block until the App is closed +session.wait() + +# 8 Evaluate your model +results = test_dataset.evaluate_classifications( + "predictions", + gt_field="ground_truth", + eval_key="eval", +) +results.print_report() +plot = results.plot_confusion_matrix() +plot.show() diff --git a/test.py b/test.py new file mode 100644 index 0000000000..18145f29aa --- /dev/null +++ b/test.py @@ -0,0 +1,115 @@ +import os +import sys +from typing import Callable, List +import itertools + +import fiftyone as fo +import fiftyone.zoo as foz + +from flash import Trainer +from flash.core.classification import FiftyOneLabels +from flash.core.data.utils import download_data +from flash.core.finetuning import NoFreeze +from flash.video import VideoClassificationData, VideoClassifier + +import torch +from torch.utils.data.sampler import RandomSampler +import kornia.augmentation as K +from pytorchvideo.transforms import ApplyTransformToKey, RandomShortSideScale, UniformTemporalSubsample +from torchvision.transforms import CenterCrop, Compose, RandomCrop, RandomHorizontalFlip + +# 1. Load your FiftyOne dataset +# Find more dataset at https://pytorchvideo.readthedocs.io/en/latest/data.html + +train_dataset = fo.Dataset.from_dir( + dataset_dir="data/kinetics/train", + dataset_type=fo.types.VideoClassificationDirectoryTree, +) + +val_dataset = fo.Dataset.from_dir( + dataset_dir="data/kinetics/val", + dataset_type=fo.types.VideoClassificationDirectoryTree, +) + +predict_dataset = fo.Dataset.from_dir( + dataset_dir="data/kinetics/predict", + dataset_type=fo.types.VideoDirectory, +) + +# 2. [Optional] Specify transforms to be used during training. +# Flash helps you to place your transform exactly where you want. +# Learn more at: +# https://lightning-flash.readthedocs.io/en/latest/general/data.html#flash.core.data.process.Preprocess +post_tensor_transform = [UniformTemporalSubsample(8), RandomShortSideScale(min_size=256, max_size=320)] +per_batch_transform_on_device = [K.Normalize(torch.tensor([0.45, 0.45, 0.45]), torch.tensor([0.225, 0.225, 0.225]))] + +train_post_tensor_transform = post_tensor_transform + [RandomCrop(244), RandomHorizontalFlip(p=0.5)] +val_post_tensor_transform = post_tensor_transform + [CenterCrop(244)] +train_per_batch_transform_on_device = per_batch_transform_on_device + +def make_transform( + post_tensor_transform: List[Callable] = post_tensor_transform, + per_batch_transform_on_device: List[Callable] = per_batch_transform_on_device +): + return { + "post_tensor_transform": Compose([ + ApplyTransformToKey( + key="video", + transform=Compose(post_tensor_transform), + ), + ]), + "per_batch_transform_on_device": Compose([ + ApplyTransformToKey( + key="video", + transform=K.VideoSequential( + *per_batch_transform_on_device, data_format="BCTHW", same_on_frame=False + ) + ), + ]), + } + + +# 2. Load the Datamodule +datamodule = VideoClassificationData.from_fiftyone_datasets( + train_dataset = train_dataset, + val_dataset = val_dataset, + predict_dataset = predict_dataset, + label_field = "ground_truth", + train_transform=make_transform(train_post_tensor_transform), + val_transform=make_transform(val_post_tensor_transform), + predict_transform=make_transform(val_post_tensor_transform), + batch_size=8, + clip_sampler="uniform", + clip_duration=1, + video_sampler=RandomSampler, + decode_audio=False, + num_workers=8 +) + +# 3. Build the model +model = VideoClassifier( + backbone="x3d_xs", + num_classes=datamodule.num_classes, + serializer=FiftyOneLabels(), + pretrained=False, +) + +# 4. Create the trainer +trainer = Trainer(fast_dev_run=True) +trainer.finetune(model, datamodule=datamodule, strategy=NoFreeze()) + +# 5. Finetune the model +trainer.finetune(model, datamodule=datamodule) + +# 6. Save it! +trainer.save_checkpoint("video_classification.pt") + +# 7. Generate predictions +predictions = trainer.predict(model, datamodule=datamodule) + +# 7b. Flatten batched predictions +predictions = list(itertools.chain.from_iterable(predictions)) + +# 8. Add predictions to dataset and analyze +predict_dataset.set_values("flash_predictions", predictions) +session = fo.launch_app(predict_dataset) From ba21d616428c475c7cd98619189e69d8fb11e15d Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Mon, 14 Jun 2021 14:20:32 -0400 Subject: [PATCH 40/54] rename from_fiftyone_datasets to from_fiftyone --- docs/source/integrations/fiftyone.rst | 2 +- flash/core/data/data_module.py | 4 ++-- .../fiftyone/image_classification_fiftyone_datasets.py | 4 ++-- tests/image/classification/test_data.py | 4 ++-- tests/image/classification/test_data_model_integration.py | 2 +- tests/image/detection/test_data.py | 6 +++--- tests/image/detection/test_data_model_integration.py | 2 +- tests/image/segmentation/test_data.py | 4 ++-- tests/video/classification/test_model.py | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst index 5747274643..fff127c890 100644 --- a/docs/source/integrations/fiftyone.rst +++ b/docs/source/integrations/fiftyone.rst @@ -69,7 +69,7 @@ ground truth annotations allowing you to perform more complex analysis with :ref:`evaluation ` of your model results. The -:meth:`~flash.core.data_module.DataModule.from_fiftyone_datasets` +:meth:`~flash.core.data_module.DataModule.from_fiftyone` method allows you to load your FiftyOne Datasets directly into a :class:`~flash.core.data_module.DataModule` to be used for training, testing, or inference. diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 72961fbca8..b8122fa507 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -1066,7 +1066,7 @@ def from_datasets( ) @classmethod - def from_fiftyone_datasets( + def from_fiftyone( cls, train_dataset: Optional[SampleCollection] = None, val_dataset: Optional[SampleCollection] = None, @@ -1122,7 +1122,7 @@ def from_fiftyone_datasets( "/path/to/dataset", dataset_type=fo.types.ImageClassificationDirectoryTree, ) - data_module = DataModule.from_fiftyone_datasets( + data_module = DataModule.from_fiftyone( train_data = train_dataset, train_transform={ "to_tensor_transform": torch.as_tensor, diff --git a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py index 40f5a61741..8b8a7b87b6 100644 --- a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py +++ b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -40,7 +40,7 @@ ) # 3 Load FiftyOne datasets -datamodule = ImageClassificationData.from_fiftyone_datasets( +datamodule = ImageClassificationData.from_fiftyone( train_dataset=train_dataset, val_dataset=val_dataset, test_dataset=test_dataset, @@ -69,7 +69,7 @@ "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" ) model.serializer = FiftyOneLabels(return_filepath=False) -datamodule = ImageClassificationData.from_fiftyone_datasets( +datamodule = ImageClassificationData.from_fiftyone( predict_dataset=test_dataset, ) predictions = trainer.predict(model, datamodule=datamodule) diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index c7d46cfc70..adb2f493a8 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -387,7 +387,7 @@ def test_from_data(data, from_function): @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") -def test_from_fiftyone_datasets(tmpdir): +def test_from_fiftyone(tmpdir): tmpdir = Path(tmpdir) (tmpdir / "a").mkdir() @@ -408,7 +408,7 @@ def test_from_fiftyone_datasets(tmpdir): s1.save() s2.save() - img_data = ImageClassificationData.from_fiftyone_datasets( + img_data = ImageClassificationData.from_fiftyone( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/classification/test_data_model_integration.py b/tests/image/classification/test_data_model_integration.py index c10b1f72c0..711bcc329f 100644 --- a/tests/image/classification/test_data_model_integration.py +++ b/tests/image/classification/test_data_model_integration.py @@ -84,7 +84,7 @@ def test_classification_fiftyone(tmpdir): s1.save() s2.save() - data = ImageClassificationData.from_fiftyone_datasets( + data = ImageClassificationData.from_fiftyone( train_dataset=train_dataset, label_field="test", batch_size=2, diff --git a/tests/image/detection/test_data.py b/tests/image/detection/test_data.py index 58ef251b18..b87ba8dec5 100644 --- a/tests/image/detection/test_data.py +++ b/tests/image/detection/test_data.py @@ -168,11 +168,11 @@ def test_image_detector_data_from_coco(tmpdir): @pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") -def test_image_detector_data_from_fiftyone_datasets(tmpdir): +def test_image_detector_data_from_fiftyone(tmpdir): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - datamodule = ObjectDetectionData.from_fiftyone_datasets(train_dataset=train_dataset, batch_size=1) + datamodule = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) data = next(iter(datamodule.train_dataloader())) imgs, labels = data[DefaultDataKeys.INPUT], data[DefaultDataKeys.TARGET] @@ -185,7 +185,7 @@ def test_image_detector_data_from_fiftyone_datasets(tmpdir): assert datamodule.val_dataloader() is None assert datamodule.test_dataloader() is None - datamodule = ObjectDetectionData.from_fiftyone_datasets( + datamodule = ObjectDetectionData.from_fiftyone( train_dataset=train_dataset, val_dataset=train_dataset, test_dataset=train_dataset, diff --git a/tests/image/detection/test_data_model_integration.py b/tests/image/detection/test_data_model_integration.py index 5697d09019..a20a4c06d3 100644 --- a/tests/image/detection/test_data_model_integration.py +++ b/tests/image/detection/test_data_model_integration.py @@ -63,7 +63,7 @@ def test_detection_fiftyone(tmpdir, model, backbone): train_dataset = _create_synth_fiftyone_dataset(tmpdir) - data = ObjectDetectionData.from_fiftyone_datasets(train_dataset=train_dataset, batch_size=1) + data = ObjectDetectionData.from_fiftyone(train_dataset=train_dataset, batch_size=1) model = ObjectDetector(model=model, backbone=backbone, num_classes=data.num_classes) trainer = flash.Trainer(fast_dev_run=True) diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index 18e81a9d84..edae4d42a4 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -252,7 +252,7 @@ def test_from_files_warning(self, tmpdir): ) @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") - def test_from_fiftyone_datasets(self, tmpdir): + def test_from_fiftyone(self, tmpdir): tmp_dir = Path(tmpdir) # create random dummy data @@ -286,7 +286,7 @@ def test_from_fiftyone_datasets(self, tmpdir): # instantiate the data module - dm = SemanticSegmentationData.from_fiftyone_datasets( + dm = SemanticSegmentationData.from_fiftyone( train_dataset=dataset, val_dataset=dataset, test_dataset=dataset, diff --git a/tests/video/classification/test_model.py b/tests/video/classification/test_model.py index 88ffd08fe2..fcb1bc68bd 100644 --- a/tests/video/classification/test_model.py +++ b/tests/video/classification/test_model.py @@ -204,7 +204,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): dir_name, dataset_type=fo.types.VideoClassificationDirectoryTree, ) - datamodule = VideoClassificationData.from_fiftyone_datasets( + datamodule = VideoClassificationData.from_fiftyone( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, @@ -243,7 +243,7 @@ def test_video_classifier_finetune_fiftyone(tmpdir): ]), } - datamodule = VideoClassificationData.from_fiftyone_datasets( + datamodule = VideoClassificationData.from_fiftyone( train_dataset=train_dataset, clip_sampler="uniform", clip_duration=half_duration, From 9a67c85a2d6cce08f6b3816f9dbd48d3c8430cd1 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Mon, 14 Jun 2021 20:04:42 -0400 Subject: [PATCH 41/54] fiftyone_visualize to visualize, update docs --- docs/source/integrations/fiftyone.rst | 78 ++++++++++++++----- flash/core/integrations/fiftyone/__init__.py | 2 +- flash/core/integrations/fiftyone/utils.py | 13 +--- .../fiftyone/image_classification.py | 9 +-- .../image_classification_fiftyone_datasets.py | 14 ++-- .../fiftyone/image_embedding.py | 51 ++++++++++++ 6 files changed, 122 insertions(+), 45 deletions(-) create mode 100644 flash_examples_integrations/fiftyone/image_embedding.py diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst index fff127c890..fe98bb810d 100644 --- a/docs/source/integrations/fiftyone.rst +++ b/docs/source/integrations/fiftyone.rst @@ -4,15 +4,15 @@ FiftyOne We have collaborated with the team at `Voxel51 `_ to integrate their tool, -`FiftyOne `_, into Flash. +`FiftyOne `_, into Lightning Flash. FiftyOne is an open-source tool for building high-quality datasets and computer vision models. The FiftyOne API and App enable you to visualize datasets and interpret models faster and more effectively. -This integration allows you to view predictions generated by Flash tasks in the +This integration allows you to view predictions generated by your tasks in the :ref:`FiftyOne App `, as well as easily incorporate -:ref:`FiftyOne Datasets ` into your Flash tasks. All image and video tasks +:ref:`FiftyOne Datasets ` into your tasks. All image and video tasks are supported! .. raw:: html @@ -35,32 +35,35 @@ In order to utilize this integration with FiftyOne, you will need to pip install fiftyone -*********************** -Visualizing predictions -*********************** +***************************** +Visualizing Flash predictions +***************************** -This integration adds the ability to visualize and explore the predictions -generated by your tasks and models by adding the -:func:`~flash.core.integrations.fiftyone.fiftyone_visualize` -function that works with the following serializers: +This section shows you how to augment your existing Lightning Flash workflows +with a couple of lines of code that let you visualize predictions in FiftyOne. +You can visualize predictions for classification, object detection, and +semantic segmentation tasks. Doing so is as easy updating your model to use +one of the following serializers: * :class:`FiftyOneLabels(return_filepath=True)` * :class:`FiftyOneSegmentationLabels(return_filepath=True)` * :class:`FiftyOneDetectionLabels(return_filepath=True)` -The :func:`~flash.core.integrations.fiftyone.fiftyone_visualize` function accepts a list of +The :func:`~flash.core.integrations.fiftyone.visualize` function then lets you visualize +your predictions in the +:ref:`FiftyOne App `. This function accepts a list of dictionaries containing :ref:`FiftyOne Label` objects -and filepaths which is the output of the FiftyOne serializers when the flag -``return_filepath=True``. +and filepaths which is the exact output of the FiftyOne serializers when the flag +``return_filepath=True`` is specified. .. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification.py :language: python :lines: 14- -************************* -Loading FiftyOne datasets -************************* +*********************** +Using FiftyOne datasets +*********************** The above workflow is great for visualizing model predictions. However, if you store your data in a FiftyOne Dataset initially, then you can also visualize @@ -69,21 +72,49 @@ ground truth annotations allowing you to perform more complex analysis with :ref:`evaluation ` of your model results. The -:meth:`~flash.core.data_module.DataModule.from_fiftyone` +:meth:`~flash.core.data.data_module.DataModule.from_fiftyone` method allows you to load your FiftyOne Datasets directly into a -:class:`~flash.core.data_module.DataModule` to be used for training, +:class:`~flash.core.data.data_module.DataModule` to be used for training, testing, or inference. .. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py :language: python :lines: 14- + +********************** +Visualizing embeddings +********************** + +FiftyOne provides the methods for +:ref:`dimensionality reduction` and +:ref:`interactive plotting`. When combined with +:ref:`embedding tasks ` in Flash, you can accomplish +powerful workflows like clustering, similarity search, pre-annotation, and more +in only a few lines of code. + +.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_embedding.py + :language: python + :lines: 14- + +.. image:: https://user-images.githubusercontent.com/21222883/121972505-45114b00-cd49-11eb-9ef5-9a69fd90bf59.png + :alt: embeddings_example + :align: center + ------ ************* API reference ************* +.. _from_fiftyone: + +DataModule.from_fiftyone +------------------------ + +.. automethod:: flash.core.data.data_module.DataModule.from_fiftyone + :noindex: + .. _fiftyone_labels: FiftyOneLabels @@ -91,7 +122,6 @@ FiftyOneLabels .. autoclass:: flash.core.classification.FiftyOneLabels :members: - :exclude-members: forward .. _fiftyone_segmentation_labels: @@ -100,7 +130,6 @@ FiftyOneSegmentationLabels .. autoclass:: flash.image.segmentation.serialization.FiftyOneSegmentationLabels :members: - :exclude-members: forward .. _fiftyone_detection_labels: @@ -109,4 +138,11 @@ FiftyOneDetectionLabels .. autoclass:: flash.image.detection.serialization.FiftyOneDetectionLabels :members: - :exclude-members: forward + + +.. _fiftyone_visualize: + +visualize +--------- + +.. autofunction:: flash.core.integrations.fiftyone.visualize diff --git a/flash/core/integrations/fiftyone/__init__.py b/flash/core/integrations/fiftyone/__init__.py index 54fcd69073..86d6bbb5b2 100644 --- a/flash/core/integrations/fiftyone/__init__.py +++ b/flash/core/integrations/fiftyone/__init__.py @@ -1 +1 @@ -from flash.core.integrations.fiftyone.utils import fiftyone_visualize, get_classes +from flash.core.integrations.fiftyone.utils import visualize, get_classes diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 9ed01f468b..6e17408842 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -2,7 +2,6 @@ from typing import Dict, List, Optional, Union import flash -from flash.core.data.data_module import DataModule from flash.core.data.data_source import DefaultDataKeys from flash.core.utilities.imports import _FIFTYONE_AVAILABLE @@ -20,10 +19,9 @@ Session = None -def fiftyone_visualize( +def visualize( labels: Union[List[Label], List[Dict[str, Label]]], filepaths: Optional[List[str]] = None, - datamodule: Optional[DataModule] = None, wait: Optional[bool] = True, label_field: Optional[str] = "predictions", **kwargs @@ -38,8 +36,6 @@ def fiftyone_visualize( filepaths and corresponding FiftyOne labels. filepaths: A list of filepaths to images or videos corresponding to the provided `labels`. - datamodule: The datamodule containing the prediction dataset used to - generate `labels`. wait: A boolean determining whether to launch the FiftyOne session and wait until the session is closed or whether to return immediately. label_field: The string of the label field in the FiftyOne dataset @@ -59,12 +55,7 @@ def fiftyone_visualize( labels = [lab["predictions"] for lab in labels] if filepaths is None: - if datamodule is None: - raise ValueError("Either `filepaths` or `datamodule` arguments are " - "required if filepaths are not provided in `labels`.") - - else: - filepaths = [s[DefaultDataKeys.FILEPATH] for s in datamodule.predict_dataset.data] + raise ValueError("The `filepaths` argument is required if filepaths are not provided in `labels`.") dataset = fo.Dataset() if filepaths: diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples_integrations/fiftyone/image_classification.py index fdcdcea87d..75a6b4f150 100644 --- a/flash_examples_integrations/fiftyone/image_classification.py +++ b/flash_examples_integrations/fiftyone/image_classification.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import itertools +from itertools import chain import flash from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data from flash.core.finetuning import FreezeUnfreeze -from flash.core.integrations.fiftyone import fiftyone_visualize +from flash.core.integrations.fiftyone import visualize from flash.image import ImageClassificationData, ImageClassifier # 1 Download data @@ -56,9 +56,8 @@ model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) -# 4b Flatten batched predictions -predictions = list(itertools.chain.from_iterable(predictions)) +predictions = list(chain.from_iterable(predictions)) # flatten batches # 5. Visualize predictions in FiftyOne # Note: this blocks until the FiftyOne App is closed -session = fiftyone_visualize(predictions) +session = visualize(predictions) diff --git a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py index 8b8a7b87b6..a32c9f0528 100644 --- a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py +++ b/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -11,13 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import itertools +from itertools import chain import flash from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data from flash.core.finetuning import FreezeUnfreeze -from flash.core.integrations.fiftyone import fiftyone_visualize +from flash.core.integrations.fiftyone import visualize from flash.image import ImageClassificationData, ImageClassifier import fiftyone as fo @@ -74,8 +74,7 @@ ) predictions = trainer.predict(model, datamodule=datamodule) -# 5b Flatten batched predictions -predictions = list(itertools.chain.from_iterable(predictions)) +predictions = list(chain.from_iterable(predictions)) # flatten batches # 6 Add predictions to dataset test_dataset.set_values("predictions", predictions) @@ -83,9 +82,6 @@ # 7 Visualize labels in the App session = fo.launch_app(test_dataset) -# 7b Block until the App is closed -session.wait() - # 8 Evaluate your model results = test_dataset.evaluate_classifications( "predictions", @@ -95,3 +91,7 @@ results.print_report() plot = results.plot_confusion_matrix() plot.show() + +# Only when running this in a script +# Block until the FiftyOne App is closed +session.wait() diff --git a/flash_examples_integrations/fiftyone/image_embedding.py b/flash_examples_integrations/fiftyone/image_embedding.py new file mode 100644 index 0000000000..b43615e546 --- /dev/null +++ b/flash_examples_integrations/fiftyone/image_embedding.py @@ -0,0 +1,51 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import torch + +import fiftyone as fo +import fiftyone.brain as fob + +from flash.core.data.utils import download_data +from flash.image import ImageEmbedder + +# 1 Download data +download_data( + "https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip" +) + +# 2 Load data into FiftyOne +dataset = fo.Dataset.from_dir( + "data/hymenoptera_data/test/", + fo.types.ImageClassificationDirectoryTree, +) + +# 3 Load model +embedder = ImageEmbedder(backbone="swav-imagenet", embedding_dim=128) + +# 4 Generate embeddings +filepaths = dataset.values("filepath") +embeddings = np.stack(embedder.predict(filepaths)) + +# 5 Visualize in FiftyOne App +results = fob.compute_visualization(dataset, embeddings=embeddings) + +session = fo.launch_app(dataset) + +plot = results.visualize(labels="ground_truth.label") +plot.show() + +# Only when running this in a script +# Block until the FiftyOne App is closed +session.wait() From 49d63bf2ac358d7ff82ed5f100a993296167f18e Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 09:45:56 -0400 Subject: [PATCH 42/54] resolve comments --- CHANGELOG.md | 5 +- docs/source/integrations/fiftyone.rst | 16 +-- flash/core/integrations/fiftyone/__init__.py | 2 +- flash/image/segmentation/serialization.py | 6 +- flash/video/classification/data.py | 64 +++++----- .../finetuning/semantic_segmentation.py | 3 - .../fiftyone/image_classification.py | 0 .../image_classification_fiftyone_datasets.py | 0 .../integrations}/fiftyone/image_embedding.py | 0 flash_examples/predict/image_embedder.py | 2 - test.py | 115 ------------------ 11 files changed, 46 insertions(+), 167 deletions(-) rename {flash_examples_integrations => flash_examples/integrations}/fiftyone/image_classification.py (100%) rename {flash_examples_integrations => flash_examples/integrations}/fiftyone/image_classification_fiftyone_datasets.py (100%) rename {flash_examples_integrations => flash_examples/integrations}/fiftyone/image_embedding.py (100%) delete mode 100644 test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bb66ee3c0e..c8e1f16c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added +- Added integration with FiftyOne ([#360](https://github.com/PyTorchLightning/lightning-flash/pull/360)) - Added support for `torch.jit` to tasks where possible and documented task JIT compatibility ([#389](https://github.com/PyTorchLightning/lightning-flash/pull/389)) - Added option to provide a `Sampler` to the `DataModule` to use when creating a `DataLoader` ([#390](https://github.com/PyTorchLightning/lightning-flash/pull/390)) - Added support for multi-label text classification and toxic comments example ([#401](https://github.com/PyTorchLightning/lightning-flash/pull/401)) @@ -18,10 +19,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Deprecated -### Added - -- Added integration with FiftyOne ([#360](https://github.com/PyTorchLightning/lightning-flash/pull/360)) - ### Fixed - Fixed a bug where the `DefaultDataKeys.METADATA` couldn't be a dict ([#393](https://github.com/PyTorchLightning/lightning-flash/pull/393)) diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst index fe98bb810d..ef681f611e 100644 --- a/docs/source/integrations/fiftyone.rst +++ b/docs/source/integrations/fiftyone.rst @@ -18,8 +18,8 @@ are supported! .. raw:: html
-
@@ -42,7 +42,7 @@ Visualizing Flash predictions This section shows you how to augment your existing Lightning Flash workflows with a couple of lines of code that let you visualize predictions in FiftyOne. You can visualize predictions for classification, object detection, and -semantic segmentation tasks. Doing so is as easy updating your model to use +semantic segmentation tasks. Doing so is as easy as updating your model to use one of the following serializers: * :class:`FiftyOneLabels(return_filepath=True)` @@ -56,7 +56,7 @@ dictionaries containing :ref:`FiftyOne Label` objects and filepaths which is the exact output of the FiftyOne serializers when the flag ``return_filepath=True`` is specified. -.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification.py +.. literalinclude:: ../../../flash_examples/integrations/fiftyone/image_classification.py :language: python :lines: 14- @@ -67,7 +67,7 @@ Using FiftyOne datasets The above workflow is great for visualizing model predictions. However, if you store your data in a FiftyOne Dataset initially, then you can also visualize -ground truth annotations allowing you to perform more complex analysis with +ground truth annotations. This allows you to perform more complex analysis with :ref:`views ` into your data and :ref:`evaluation ` of your model results. @@ -77,7 +77,7 @@ method allows you to load your FiftyOne Datasets directly into a :class:`~flash.core.data.data_module.DataModule` to be used for training, testing, or inference. -.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py +.. literalinclude:: ../../../flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py :language: python :lines: 14- @@ -93,11 +93,11 @@ FiftyOne provides the methods for powerful workflows like clustering, similarity search, pre-annotation, and more in only a few lines of code. -.. literalinclude:: ../../../flash_examples_integrations/fiftyone/image_embedding.py +.. literalinclude:: ../../../flash_examples/integrations/fiftyone/image_embedding.py :language: python :lines: 14- -.. image:: https://user-images.githubusercontent.com/21222883/121972505-45114b00-cd49-11eb-9ef5-9a69fd90bf59.png +.. image:: https://pl-flash-data.s3.amazonaws.com/assets/fiftyone/embeddings.png :alt: embeddings_example :align: center diff --git a/flash/core/integrations/fiftyone/__init__.py b/flash/core/integrations/fiftyone/__init__.py index 86d6bbb5b2..cc7b22cbb7 100644 --- a/flash/core/integrations/fiftyone/__init__.py +++ b/flash/core/integrations/fiftyone/__init__.py @@ -1 +1 @@ -from flash.core.integrations.fiftyone.utils import visualize, get_classes +from flash.core.integrations.fiftyone.utils import visualize diff --git a/flash/image/segmentation/serialization.py b/flash/image/segmentation/serialization.py index 5a342f8b55..15fd73f72c 100644 --- a/flash/image/segmentation/serialization.py +++ b/flash/image/segmentation/serialization.py @@ -84,8 +84,7 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> torch.Tensor: labels_vis = K.utils.tensor_to_image(labels_vis) plt.imshow(labels_vis) plt.show() -<<<<<<< HEAD - return labels + return labels.tolist() class FiftyOneSegmentationLabels(SegmentationLabels): @@ -120,6 +119,3 @@ def serialize(self, sample: Dict[str, torch.Tensor]) -> Union[Segmentation, Dict return {"filepath": filepath, "predictions": fo_predictions} else: return fo_predictions -======= - return labels.tolist() ->>>>>>> upstream/master diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 78711e6cd2..dc755f1fe7 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import pathlib -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Type, Union +from typing import Any, Callable, Dict, List, Optional, Type, Union import numpy as np import torch @@ -34,8 +34,6 @@ if _FIFTYONE_AVAILABLE: from fiftyone.core.collections import SampleCollection from fiftyone.core.labels import Classification - - from flash.core.integrations.fiftyone import get_classes else: Classification, SampleCollection = None, None @@ -74,6 +72,20 @@ def __init__( self.decode_audio = decode_audio self.decoder = decoder + def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': + ds = self._make_encoded_video_dataset(data) + if self.training: + label_to_class_mapping = {p[1]: p[0].split("/")[-2] for p in ds._labeled_videos._paths_and_labels} + self.set_state(LabelsState(label_to_class_mapping)) + dataset.num_classes = len(np.unique([s[1]['label'] for s in ds._labeled_videos])) + return ds + + def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: + video_path = sample[DefaultDataKeys.INPUT] + sample.update(self._encoded_video_to_dict(EncodedVideo.from_path(video_path))) + sample[DefaultDataKeys.METADATA] = {"filepath": video_path} + return sample + def _encoded_video_to_dict(self, video) -> Dict[str, Any]: ( clip_start, @@ -107,8 +119,13 @@ def _encoded_video_to_dict(self, video) -> Dict[str, Any]: } if audio_samples is not None else {}), } + def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': + raise NotImplementedError( + "Subclass must implement _make_encoded_video_dataset()" + ) + -class VideoClassificationPathsDataSource(PathsDataSource, BaseVideoClassification): +class VideoClassificationPathsDataSource(BaseVideoClassification, PathsDataSource): def __init__( self, @@ -117,14 +134,16 @@ def __init__( decode_audio: bool = True, decoder: str = "pyav", ): - super().__init__(extensions=("mp4", "avi")) - BaseVideoClassification.__init__( - self, + super().__init__( clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, decoder=decoder, ) + PathsDataSource.__init__( + self, + extensions=("mp4", "avi"), + ) def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': ds: EncodedVideoDataset = labeled_encoded_video_dataset( @@ -136,22 +155,11 @@ def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': ) return ds - def load_data(self, data: str, dataset: Optional[Any] = None) -> 'EncodedVideoDataset': - ds = self._make_encoded_video_dataset(data) - if self.training: - label_to_class_mapping = {p[1]: p[0].split("/")[-2] for p in ds._labeled_videos._paths_and_labels} - self.set_state(LabelsState(label_to_class_mapping)) - dataset.num_classes = len(np.unique([s[1]['label'] for s in ds._labeled_videos])) - return ds - - def predict_load_sample(self, sample: Dict[str, Any]) -> Dict[str, Any]: - video_path = sample[DefaultDataKeys.INPUT] - sample.update(self._encoded_video_to_dict(EncodedVideo.from_path(video_path))) - sample[DefaultDataKeys.METADATA] = {"filepath": video_path} - return sample - -class VideoClassificationFiftyOneDataSource(VideoClassificationPathsDataSource): +class VideoClassificationFiftyOneDataSource( + BaseVideoClassification, + FiftyOneDataSource, +): def __init__( self, @@ -160,23 +168,24 @@ def __init__( decode_audio: bool = True, decoder: str = "pyav", label_field: str = "ground_truth", - **kwargs ): super().__init__( clip_sampler=clip_sampler, video_sampler=video_sampler, decode_audio=decode_audio, decoder=decoder, - **kwargs ) - self.label_field = label_field + FiftyOneDataSource.__init__( + self, + label_field=label_field, + ) @property def label_cls(self): return Classification def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDataset': - classes = get_classes(data, self.label_field) + classes = self._get_classes(data) label_to_class_mapping = dict(enumerate(classes)) class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} @@ -193,9 +202,6 @@ def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDa ) return ds - def predict_load_data(self, data: SampleCollection, dataset: Optional[Any] = None) -> Sequence[Mapping[str, Any]]: - return [{DefaultDataKeys.INPUT: f} for f in data.values("filepath")] - class VideoClassificationPreprocess(Preprocess): diff --git a/flash_examples/finetuning/semantic_segmentation.py b/flash_examples/finetuning/semantic_segmentation.py index 36a8db8745..049bd9d318 100644 --- a/flash_examples/finetuning/semantic_segmentation.py +++ b/flash_examples/finetuning/semantic_segmentation.py @@ -1,6 +1,3 @@ -import pytorch_lightning as pl - -pl.seed_everything(42) # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/flash_examples_integrations/fiftyone/image_classification.py b/flash_examples/integrations/fiftyone/image_classification.py similarity index 100% rename from flash_examples_integrations/fiftyone/image_classification.py rename to flash_examples/integrations/fiftyone/image_classification.py diff --git a/flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py similarity index 100% rename from flash_examples_integrations/fiftyone/image_classification_fiftyone_datasets.py rename to flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py diff --git a/flash_examples_integrations/fiftyone/image_embedding.py b/flash_examples/integrations/fiftyone/image_embedding.py similarity index 100% rename from flash_examples_integrations/fiftyone/image_embedding.py rename to flash_examples/integrations/fiftyone/image_embedding.py diff --git a/flash_examples/predict/image_embedder.py b/flash_examples/predict/image_embedder.py index 1c99853dcf..d763766c04 100644 --- a/flash_examples/predict/image_embedder.py +++ b/flash_examples/predict/image_embedder.py @@ -1,5 +1,3 @@ -import pytorch_lightning as pl -pl.seed_everything(42) # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test.py b/test.py deleted file mode 100644 index 18145f29aa..0000000000 --- a/test.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import sys -from typing import Callable, List -import itertools - -import fiftyone as fo -import fiftyone.zoo as foz - -from flash import Trainer -from flash.core.classification import FiftyOneLabels -from flash.core.data.utils import download_data -from flash.core.finetuning import NoFreeze -from flash.video import VideoClassificationData, VideoClassifier - -import torch -from torch.utils.data.sampler import RandomSampler -import kornia.augmentation as K -from pytorchvideo.transforms import ApplyTransformToKey, RandomShortSideScale, UniformTemporalSubsample -from torchvision.transforms import CenterCrop, Compose, RandomCrop, RandomHorizontalFlip - -# 1. Load your FiftyOne dataset -# Find more dataset at https://pytorchvideo.readthedocs.io/en/latest/data.html - -train_dataset = fo.Dataset.from_dir( - dataset_dir="data/kinetics/train", - dataset_type=fo.types.VideoClassificationDirectoryTree, -) - -val_dataset = fo.Dataset.from_dir( - dataset_dir="data/kinetics/val", - dataset_type=fo.types.VideoClassificationDirectoryTree, -) - -predict_dataset = fo.Dataset.from_dir( - dataset_dir="data/kinetics/predict", - dataset_type=fo.types.VideoDirectory, -) - -# 2. [Optional] Specify transforms to be used during training. -# Flash helps you to place your transform exactly where you want. -# Learn more at: -# https://lightning-flash.readthedocs.io/en/latest/general/data.html#flash.core.data.process.Preprocess -post_tensor_transform = [UniformTemporalSubsample(8), RandomShortSideScale(min_size=256, max_size=320)] -per_batch_transform_on_device = [K.Normalize(torch.tensor([0.45, 0.45, 0.45]), torch.tensor([0.225, 0.225, 0.225]))] - -train_post_tensor_transform = post_tensor_transform + [RandomCrop(244), RandomHorizontalFlip(p=0.5)] -val_post_tensor_transform = post_tensor_transform + [CenterCrop(244)] -train_per_batch_transform_on_device = per_batch_transform_on_device - -def make_transform( - post_tensor_transform: List[Callable] = post_tensor_transform, - per_batch_transform_on_device: List[Callable] = per_batch_transform_on_device -): - return { - "post_tensor_transform": Compose([ - ApplyTransformToKey( - key="video", - transform=Compose(post_tensor_transform), - ), - ]), - "per_batch_transform_on_device": Compose([ - ApplyTransformToKey( - key="video", - transform=K.VideoSequential( - *per_batch_transform_on_device, data_format="BCTHW", same_on_frame=False - ) - ), - ]), - } - - -# 2. Load the Datamodule -datamodule = VideoClassificationData.from_fiftyone_datasets( - train_dataset = train_dataset, - val_dataset = val_dataset, - predict_dataset = predict_dataset, - label_field = "ground_truth", - train_transform=make_transform(train_post_tensor_transform), - val_transform=make_transform(val_post_tensor_transform), - predict_transform=make_transform(val_post_tensor_transform), - batch_size=8, - clip_sampler="uniform", - clip_duration=1, - video_sampler=RandomSampler, - decode_audio=False, - num_workers=8 -) - -# 3. Build the model -model = VideoClassifier( - backbone="x3d_xs", - num_classes=datamodule.num_classes, - serializer=FiftyOneLabels(), - pretrained=False, -) - -# 4. Create the trainer -trainer = Trainer(fast_dev_run=True) -trainer.finetune(model, datamodule=datamodule, strategy=NoFreeze()) - -# 5. Finetune the model -trainer.finetune(model, datamodule=datamodule) - -# 6. Save it! -trainer.save_checkpoint("video_classification.pt") - -# 7. Generate predictions -predictions = trainer.predict(model, datamodule=datamodule) - -# 7b. Flatten batched predictions -predictions = list(itertools.chain.from_iterable(predictions)) - -# 8. Add predictions to dataset and analyze -predict_dataset.set_values("flash_predictions", predictions) -session = fo.launch_app(predict_dataset) From 427d280201f0a067901cc991805d580854ffc96c Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 10:33:30 -0400 Subject: [PATCH 43/54] resolve test issues --- docs/source/template/data.rst | 2 +- flash/image/segmentation/serialization.py | 21 +++++++++++---------- tests/core/test_integrations.py | 2 +- tests/examples/test_integrations.py | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/source/template/data.rst b/docs/source/template/data.rst index 92ee56ae28..8b8881ce24 100644 --- a/docs/source/template/data.rst +++ b/docs/source/template/data.rst @@ -85,7 +85,7 @@ Here's how it looks (from `video/classification.data.py torch.Tensor: class FiftyOneSegmentationLabels(SegmentationLabels): - """A :class:`.Serializer` which converts the model outputs to FiftyOne segmentation format. - - Args: - labels_map: A dictionary that map the labels ids to pixel intensities. - visualize: Wether to visualize the image labels. - return_filepath: Boolean determining whether to return a dict - containing filepath and FiftyOne labels (True) or only a - list of FiftyOne labels (False) - """ def __init__( self, @@ -104,6 +96,15 @@ def __init__( visualize: bool = False, return_filepath: bool = False, ): + """A :class:`.Serializer` which converts the model outputs to FiftyOne segmentation format. + + Args: + labels_map: A dictionary that map the labels ids to pixel intensities. + visualize: Wether to visualize the image labels. + return_filepath: Boolean determining whether to return a dict + containing filepath and FiftyOne labels (True) or only a + list of FiftyOne labels (False) + """ if not _FIFTYONE_AVAILABLE: raise ModuleNotFoundError("Please, run `pip install fiftyone`.") @@ -113,7 +114,7 @@ def __init__( def serialize(self, sample: Dict[str, torch.Tensor]) -> Union[Segmentation, Dict[str, Any]]: labels = super().serialize(sample) - fo_predictions = Segmentation(mask=labels.numpy()) + fo_predictions = Segmentation(mask=np.array(labels)) if self.return_filepath: filepath = sample[DefaultDataKeys.METADATA]["filepath"] return {"filepath": filepath, "predictions": fo_predictions} diff --git a/tests/core/test_integrations.py b/tests/core/test_integrations.py index 364de1ddf1..1dea0a9a81 100644 --- a/tests/core/test_integrations.py +++ b/tests/core/test_integrations.py @@ -37,4 +37,4 @@ ] ) def test_integrations(tmpdir, folder, file): - run_test(str(root / "flash_examples_integrations" / folder / file)) + run_test(str(root / "flash_examples" / "integrations" / folder / file)) diff --git a/tests/examples/test_integrations.py b/tests/examples/test_integrations.py index 364de1ddf1..1dea0a9a81 100644 --- a/tests/examples/test_integrations.py +++ b/tests/examples/test_integrations.py @@ -37,4 +37,4 @@ ] ) def test_integrations(tmpdir, folder, file): - run_test(str(root / "flash_examples_integrations" / folder / file)) + run_test(str(root / "flash_examples" / "integrations" / folder / file)) From 8fe7d7603d0067bda65b080024f5cc7b8c59eb55 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 10:41:58 -0400 Subject: [PATCH 44/54] formatting --- docs/source/integrations/fiftyone.rst | 20 +++++++++---------- .../fiftyone/image_classification.py | 14 ++++++------- .../image_classification_fiftyone_datasets.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/source/integrations/fiftyone.rst b/docs/source/integrations/fiftyone.rst index ef681f611e..d12392e5ec 100644 --- a/docs/source/integrations/fiftyone.rst +++ b/docs/source/integrations/fiftyone.rst @@ -2,7 +2,7 @@ FiftyOne ######## -We have collaborated with the team at +We have collaborated with the team at `Voxel51 `_ to integrate their tool, `FiftyOne `_, into Lightning Flash. @@ -11,7 +11,7 @@ datasets and computer vision models. The FiftyOne API and App enable you to visualize datasets and interpret models faster and more effectively. This integration allows you to view predictions generated by your tasks in the -:ref:`FiftyOne App `, as well as easily incorporate +:ref:`FiftyOne App `, as well as easily incorporate :ref:`FiftyOne Datasets ` into your tasks. All image and video tasks are supported! @@ -27,11 +27,11 @@ are supported! Installation ************ -In order to utilize this integration with FiftyOne, you will need to +In order to utilize this integration with FiftyOne, you will need to :ref:`install the tool`: .. code:: shell - + pip install fiftyone @@ -40,9 +40,9 @@ Visualizing Flash predictions ***************************** This section shows you how to augment your existing Lightning Flash workflows -with a couple of lines of code that let you visualize predictions in FiftyOne. +with a couple of lines of code that let you visualize predictions in FiftyOne. You can visualize predictions for classification, object detection, and -semantic segmentation tasks. Doing so is as easy as updating your model to use +semantic segmentation tasks. Doing so is as easy as updating your model to use one of the following serializers: * :class:`FiftyOneLabels(return_filepath=True)` @@ -50,8 +50,8 @@ one of the following serializers: * :class:`FiftyOneDetectionLabels(return_filepath=True)` The :func:`~flash.core.integrations.fiftyone.visualize` function then lets you visualize -your predictions in the -:ref:`FiftyOne App `. This function accepts a list of +your predictions in the +:ref:`FiftyOne App `. This function accepts a list of dictionaries containing :ref:`FiftyOne Label` objects and filepaths which is the exact output of the FiftyOne serializers when the flag ``return_filepath=True`` is specified. @@ -67,7 +67,7 @@ Using FiftyOne datasets The above workflow is great for visualizing model predictions. However, if you store your data in a FiftyOne Dataset initially, then you can also visualize -ground truth annotations. This allows you to perform more complex analysis with +ground truth annotations. This allows you to perform more complex analysis with :ref:`views ` into your data and :ref:`evaluation ` of your model results. @@ -86,7 +86,7 @@ testing, or inference. Visualizing embeddings ********************** -FiftyOne provides the methods for +FiftyOne provides the methods for :ref:`dimensionality reduction` and :ref:`interactive plotting`. When combined with :ref:`embedding tasks ` in Flash, you can accomplish diff --git a/flash_examples/integrations/fiftyone/image_classification.py b/flash_examples/integrations/fiftyone/image_classification.py index 75a6b4f150..bb81cd07d2 100644 --- a/flash_examples/integrations/fiftyone/image_classification.py +++ b/flash_examples/integrations/fiftyone/image_classification.py @@ -33,18 +33,18 @@ # 3 Fine tune a model model = ImageClassifier( - backbone="resnet18", - num_classes=datamodule.num_classes, + backbone="resnet18", + num_classes=datamodule.num_classes, serializer=Labels(), ) trainer = flash.Trainer( - max_epochs=1, - limit_train_batches=1, + max_epochs=1, + limit_train_batches=1, limit_val_batches=1, ) trainer.finetune( - model, - datamodule=datamodule, + model, + datamodule=datamodule, strategy=FreezeUnfreeze(unfreeze_epoch=1), ) trainer.save_checkpoint("image_classification_model.pt") @@ -56,7 +56,7 @@ model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) -predictions = list(chain.from_iterable(predictions)) # flatten batches +predictions = list(chain.from_iterable(predictions)) # flatten batches # 5. Visualize predictions in FiftyOne # Note: this blocks until the FiftyOne App is closed diff --git a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py index a32c9f0528..0fda1358f5 100644 --- a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py +++ b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -74,7 +74,7 @@ ) predictions = trainer.predict(model, datamodule=datamodule) -predictions = list(chain.from_iterable(predictions)) # flatten batches +predictions = list(chain.from_iterable(predictions)) # flatten batches # 6 Add predictions to dataset test_dataset.set_values("predictions", predictions) From a908b9d90cbaa9fa2a34760195a2115b1eb06be6 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 10:46:19 -0400 Subject: [PATCH 45/54] formatting --- .../fiftyone/image_classification_fiftyone_datasets.py | 4 ++-- flash_examples/integrations/fiftyone/image_embedding.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py index 0fda1358f5..fd03abc414 100644 --- a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py +++ b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -13,6 +13,8 @@ # limitations under the License. from itertools import chain +import fiftyone as fo + import flash from flash.core.classification import FiftyOneLabels, Labels, Probabilities from flash.core.data.utils import download_data @@ -20,8 +22,6 @@ from flash.core.integrations.fiftyone import visualize from flash.image import ImageClassificationData, ImageClassifier -import fiftyone as fo - # 1 Download data download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") diff --git a/flash_examples/integrations/fiftyone/image_embedding.py b/flash_examples/integrations/fiftyone/image_embedding.py index b43615e546..6c820fb34e 100644 --- a/flash_examples/integrations/fiftyone/image_embedding.py +++ b/flash_examples/integrations/fiftyone/image_embedding.py @@ -11,11 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np -import torch - import fiftyone as fo import fiftyone.brain as fob +import numpy as np +import torch from flash.core.data.utils import download_data from flash.image import ImageEmbedder From 895e0bff9429877b7121db327b7c02e47b7a7ec9 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 10:51:10 -0400 Subject: [PATCH 46/54] yapf formatting --- flash/core/data/data_module.py | 4 ++-- flash/image/detection/serialization.py | 3 +-- flash/video/classification/data.py | 8 +++----- .../integrations/fiftyone/image_classification.py | 4 +--- .../fiftyone/image_classification_fiftyone_datasets.py | 8 ++------ flash_examples/integrations/fiftyone/image_embedding.py | 4 +--- 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 9196ad9f93..33ed2b020d 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -322,8 +322,8 @@ def _predict_dataloader(self) -> DataLoader: @property def num_classes(self) -> Optional[int]: return ( - getattr(self.train_dataset, "num_classes", None) or getattr(self.val_dataset, "num_classes", None) or - getattr(self.test_dataset, "num_classes", None) + getattr(self.train_dataset, "num_classes", None) or getattr(self.val_dataset, "num_classes", None) + or getattr(self.test_dataset, "num_classes", None) ) @property diff --git a/flash/image/detection/serialization.py b/flash/image/detection/serialization.py index 73d28d8b96..4d6bbe6ea7 100644 --- a/flash/image/detection/serialization.py +++ b/flash/image/detection/serialization.py @@ -64,8 +64,7 @@ def __init__( def serialize(self, sample: Dict[str, Any]) -> Union[Detections, Dict[str, Any]]: if DefaultDataKeys.METADATA not in sample: - raise ValueError("sample requires DefaultDataKeys.METADATA to use " - "a FiftyOneDetectionLabels serializer.") + raise ValueError("sample requires DefaultDataKeys.METADATA to use a FiftyOneDetectionLabels serializer.") labels = None if self._labels is not None: diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index dc755f1fe7..2a6c09e3a5 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -120,9 +120,7 @@ def _encoded_video_to_dict(self, video) -> Dict[str, Any]: } def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': - raise NotImplementedError( - "Subclass must implement _make_encoded_video_dataset()" - ) + raise NotImplementedError("Subclass must implement _make_encoded_video_dataset()") class VideoClassificationPathsDataSource(BaseVideoClassification, PathsDataSource): @@ -157,8 +155,8 @@ def _make_encoded_video_dataset(self, data) -> 'EncodedVideoDataset': class VideoClassificationFiftyOneDataSource( - BaseVideoClassification, - FiftyOneDataSource, + BaseVideoClassification, + FiftyOneDataSource, ): def __init__( diff --git a/flash_examples/integrations/fiftyone/image_classification.py b/flash_examples/integrations/fiftyone/image_classification.py index bb81cd07d2..2b9e13717b 100644 --- a/flash_examples/integrations/fiftyone/image_classification.py +++ b/flash_examples/integrations/fiftyone/image_classification.py @@ -50,9 +50,7 @@ trainer.save_checkpoint("image_classification_model.pt") # 4 Predict from checkpoint -model = ImageClassifier.load_from_checkpoint( - "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" -) +model = ImageClassifier.load_from_checkpoint("https://flash-weights.s3.amazonaws.com/image_classification_model.pt") model.serializer = FiftyOneLabels(return_filepath=True) predictions = trainer.predict(model, datamodule=datamodule) diff --git a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py index fd03abc414..d7bc4cb72a 100644 --- a/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py +++ b/flash_examples/integrations/fiftyone/image_classification_fiftyone_datasets.py @@ -65,13 +65,9 @@ trainer.save_checkpoint("image_classification_model.pt") # 5 Predict from checkpoint on data with ground truth -model = ImageClassifier.load_from_checkpoint( - "https://flash-weights.s3.amazonaws.com/image_classification_model.pt" -) +model = ImageClassifier.load_from_checkpoint("https://flash-weights.s3.amazonaws.com/image_classification_model.pt") model.serializer = FiftyOneLabels(return_filepath=False) -datamodule = ImageClassificationData.from_fiftyone( - predict_dataset=test_dataset, -) +datamodule = ImageClassificationData.from_fiftyone(predict_dataset=test_dataset) predictions = trainer.predict(model, datamodule=datamodule) predictions = list(chain.from_iterable(predictions)) # flatten batches diff --git a/flash_examples/integrations/fiftyone/image_embedding.py b/flash_examples/integrations/fiftyone/image_embedding.py index 6c820fb34e..71b9ef68e3 100644 --- a/flash_examples/integrations/fiftyone/image_embedding.py +++ b/flash_examples/integrations/fiftyone/image_embedding.py @@ -20,9 +20,7 @@ from flash.image import ImageEmbedder # 1 Download data -download_data( - "https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip" -) +download_data("https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip") # 2 Load data into FiftyOne dataset = fo.Dataset.from_dir( From 45dded667d319001c93db0445f2ce9f863c883ab Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 11:22:10 -0400 Subject: [PATCH 47/54] update for current FO version --- tests/image/segmentation/test_data.py | 30 ++++++++++++--------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index edae4d42a4..2c95b3715e 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -257,33 +257,29 @@ def test_from_fiftyone(self, tmpdir): # create random dummy data - os.makedirs(str(tmp_dir / "images")) - os.makedirs(str(tmp_dir / "targets")) - images = [ - str(tmp_dir / "images" / "img1.png"), - str(tmp_dir / "images" / "img2.png"), - str(tmp_dir / "images" / "img3.png"), - ] - - targets = [ - str(tmp_dir / "targets" / "img1.png"), - str(tmp_dir / "targets" / "img2.png"), - str(tmp_dir / "targets" / "img3.png"), + str(tmp_dir / "img1.png"), + str(tmp_dir / "img2.png"), + str(tmp_dir / "img3.png"), ] num_classes: int = 2 img_size: Tuple[int, int] = (196, 196) - create_random_data(images, targets, img_size, num_classes) + + for img_file in images: + _rand_image(img_size).save(img_file) + + targets = [np.array(_rand_labels(img_size, num_classes)) for _ in range(3)] dataset = fo.Dataset.from_dir( str(tmp_dir), - dataset_type=fo.types.ImageSegmentationDirectory, - data_path="images", - labels_path="targets", - force_grayscale=True, + dataset_type=fo.types.ImageDirectory, ) + for idx, sample in enumerate(dataset): + sample["ground_truth"] = fo.Segmentation(mask=targets[idx][:, :, 0]) + sample.save() + # instantiate the data module dm = SemanticSegmentationData.from_fiftyone( From 60c394739cce246fc89c6ee85f72ce4a571511c0 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 11:59:19 -0400 Subject: [PATCH 48/54] resolve metadata batch issue --- flash/core/data/batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/core/data/batch.py b/flash/core/data/batch.py index cf524d2cef..587207a0a0 100644 --- a/flash/core/data/batch.py +++ b/flash/core/data/batch.py @@ -230,7 +230,7 @@ def forward(self, samples: Sequence[Any]) -> Any: with self._collate_context: samples, metadata = self._extract_metadata(samples) samples = self.collate_fn(samples) - if metadata: + if metadata and isinstance(samples, dict): samples[DefaultDataKeys.METADATA] = metadata self.callback.on_collate(samples, self.stage) From 5120209476df98272c00f7797b70cd4c4cb3843c Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 13:47:54 -0400 Subject: [PATCH 49/54] use current FO release, update test requirements --- flash/video/classification/data.py | 3 ++- tests/image/classification/test_data.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index 2a6c09e3a5..afab163bdd 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -187,7 +187,8 @@ def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDa label_to_class_mapping = dict(enumerate(classes)) class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} - filepaths, labels = data.values(["filepath", self.label_field + ".label"]) + filepaths = data.values("filepath") + labels = data.values(self.label_field + ".label"]) targets = [class_to_label_mapping[lab] for lab in labels] labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index adb2f493a8..146720cd44 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -37,6 +37,7 @@ def _dummy_image_loader(_): return torch.rand(3, 196, 196) +@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed.") def _rand_image(size: Tuple[int, int] = None): if size is None: _size = np.random.choice([196, 244]) From bec64618bd7f8f723655545165e521be6e09a2a7 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 13:59:51 -0400 Subject: [PATCH 50/54] syntax --- flash/video/classification/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py index afab163bdd..c75d575784 100644 --- a/flash/video/classification/data.py +++ b/flash/video/classification/data.py @@ -188,7 +188,7 @@ def _make_encoded_video_dataset(self, data: SampleCollection) -> 'EncodedVideoDa class_to_label_mapping = {c: lab for lab, c in label_to_class_mapping.items()} filepaths = data.values("filepath") - labels = data.values(self.label_field + ".label"]) + labels = data.values(self.label_field + ".label") targets = [class_to_label_mapping[lab] for lab in labels] labeled_video_paths = LabeledVideoPaths(list(zip(filepaths, targets))) From 6eef60eaac9f2474de620fae82974d4b0a125758 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 14:32:04 -0400 Subject: [PATCH 51/54] update test --- tests/image/classification/test_data.py | 31 +++++++++++++++++++------ tests/image/segmentation/test_data.py | 6 +++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tests/image/classification/test_data.py b/tests/image/classification/test_data.py index 146720cd44..d16a9c0d0a 100644 --- a/tests/image/classification/test_data.py +++ b/tests/image/classification/test_data.py @@ -37,7 +37,6 @@ def _dummy_image_loader(_): return torch.rand(3, 196, 196) -@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed.") def _rand_image(size: Tuple[int, int] = None): if size is None: _size = np.random.choice([196, 244]) @@ -387,6 +386,7 @@ def test_from_data(data, from_function): assert list(labels.numpy()) == [2, 5] +@pytest.mark.skipif(not _IMAGE_AVAILABLE, reason="image libraries aren't installed.") @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.") def test_from_fiftyone(tmpdir): tmpdir = Path(tmpdir) @@ -401,26 +401,43 @@ def test_from_fiftyone(tmpdir): str(tmpdir / "b_1.png"), ] - train_dataset = fo.Dataset.from_dir(str(tmpdir), dataset_type=fo.types.ImageDirectory) - s1 = train_dataset[train_images[0]] - s2 = train_dataset[train_images[1]] + dataset = fo.Dataset.from_dir(str(tmpdir), dataset_type=fo.types.ImageDirectory) + s1 = dataset[train_images[0]] + s2 = dataset[train_images[1]] s1["test"] = fo.Classification(label="1") s2["test"] = fo.Classification(label="2") s1.save() s2.save() img_data = ImageClassificationData.from_fiftyone( - train_dataset=train_dataset, + train_dataset=dataset, + test_dataset=dataset, + val_dataset=dataset, label_field="test", batch_size=2, num_workers=0, ) assert img_data.train_dataloader() is not None - assert img_data.val_dataloader() is None - assert img_data.test_dataloader() is None + assert img_data.val_dataloader() is not None + assert img_data.test_dataloader() is not None + # check train data data = next(iter(img_data.train_dataloader())) imgs, labels = data['input'], data['target'] assert imgs.shape == (2, 3, 196, 196) assert labels.shape == (2, ) assert sorted(list(labels.numpy())) == [0, 1] + + # check val data + data = next(iter(img_data.val_dataloader())) + imgs, labels = data['input'], data['target'] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, ) + assert sorted(list(labels.numpy())) == [0, 1] + + # check test data + data = next(iter(img_data.test_dataloader())) + imgs, labels = data['input'], data['target'] + assert imgs.shape == (2, 3, 196, 196) + assert labels.shape == (2, ) + assert sorted(list(labels.numpy())) == [0, 1] diff --git a/tests/image/segmentation/test_data.py b/tests/image/segmentation/test_data.py index 2c95b3715e..089871fedb 100644 --- a/tests/image/segmentation/test_data.py +++ b/tests/image/segmentation/test_data.py @@ -286,6 +286,7 @@ def test_from_fiftyone(self, tmpdir): train_dataset=dataset, val_dataset=dataset, test_dataset=dataset, + predict_dataset=dataset, batch_size=2, num_workers=0, num_classes=num_classes, @@ -313,6 +314,11 @@ def test_from_fiftyone(self, tmpdir): assert imgs.shape == (2, 3, 196, 196) assert labels.shape == (2, 196, 196) + # check predict data + data = next(iter(dm.predict_dataloader())) + imgs = data[DefaultDataKeys.INPUT] + assert imgs.shape == (2, 3, 196, 196) + def test_map_labels(self, tmpdir): tmp_dir = Path(tmpdir) From 9c9b321505de7d6983b86fb28f916e3342968512 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 14:58:17 -0400 Subject: [PATCH 52/54] update tests --- flash/core/integrations/fiftyone/utils.py | 13 ------------- tests/core/test_classification.py | 7 +++++++ tests/image/detection/test_serialization.py | 6 ++++++ .../image/segmentation/test_serialization.py | 19 +++++++++++++++---- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/flash/core/integrations/fiftyone/utils.py b/flash/core/integrations/fiftyone/utils.py index 6e17408842..5498a801ed 100644 --- a/flash/core/integrations/fiftyone/utils.py +++ b/flash/core/integrations/fiftyone/utils.py @@ -68,16 +68,3 @@ def visualize( if wait: session.wait() return session - - -def get_classes(data, label_field: str): - classes = data.classes.get(label_field, None) - - if not classes: - classes = data.default_classes - - if not classes: - label_path = data._get_label_field_path(label_field, "label")[1] - classes = data.distinct(label_path) - - return classes diff --git a/tests/core/test_classification.py b/tests/core/test_classification.py index be42e661e9..9281c36ab4 100644 --- a/tests/core/test_classification.py +++ b/tests/core/test_classification.py @@ -49,6 +49,13 @@ def test_classification_serializers_fiftyone(): example_output = {DefaultDataKeys.PREDS: logits, DefaultDataKeys.METADATA: {"filepath": "something"}} # 3 classes labels = ['class_1', 'class_2', 'class_3'] + predictions = FiftyOneLabels(return_filepath=True).serialize(example_output) + assert predictions["predictions"].label == '2' + assert predictions["filepath"] == "something" + predictions = FiftyOneLabels(labels, return_filepath=True).serialize(example_output) + assert predictions["predictions"].label == 'class_3' + assert predictions["filepath"] == "something" + predictions = FiftyOneLabels(store_logits=True).serialize(example_output) assert torch.allclose(torch.tensor(predictions.logits), logits) assert torch.allclose(torch.tensor(predictions.confidence), torch.softmax(logits, -1)[-1]) diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py index fc600bef6f..9e1080a980 100644 --- a/tests/image/detection/test_serialization.py +++ b/tests/image/detection/test_serialization.py @@ -15,6 +15,7 @@ def test_smoke(self): def test_serialize_fiftyone(self): serial = FiftyOneDetectionLabels() + filepath_serial = FiftyOneDetectionLabels(return_filepath=True) sample = { DefaultDataKeys.PREDS: [ @@ -34,3 +35,8 @@ def test_serialize_fiftyone(self): detections = serial.serialize(sample) assert len(detections.detections) == 1 assert detections.detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] + + detections = filepath_serial.serialize(sample) + assert len(detections["predictions"].detections) == 1 + assert detections["predictions"].detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] + assert detections["filepath"] == "something" diff --git a/tests/image/segmentation/test_serialization.py b/tests/image/segmentation/test_serialization.py index 182d914acb..76fff0918a 100644 --- a/tests/image/segmentation/test_serialization.py +++ b/tests/image/segmentation/test_serialization.py @@ -39,15 +39,26 @@ def test_serialize(self): @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone is not installed for testing") def test_serialize_fiftyone(self): serial = FiftyOneSegmentationLabels() + filepath_serial = FiftyOneSegmentationLabels(return_filepath=True) - sample = torch.zeros(5, 2, 3) - sample[1, 1, 2] = 1 # add peak in class 2 - sample[3, 0, 1] = 1 # add peak in class 4 + preds = torch.zeros(5, 2, 3) + preds[1, 1, 2] = 1 # add peak in class 2 + preds[3, 0, 1] = 1 # add peak in class 4 - segmentation = serial.serialize({DefaultDataKeys.PREDS: sample}) + sample = { + DefaultDataKeys.PREDS: preds, + DefaultDataKeys.METADATA: {"filepath": "something"}, + } + + segmentation = serial.serialize(sample) assert segmentation.mask[1, 2] == 1 assert segmentation.mask[0, 1] == 3 + segmentation = filepath_serial.serialize(sample) + assert segmentation["predictions"].mask[1, 2] == 1 + assert segmentation["predictions"].mask[0, 1] == 3 + assert segmentation["filepath"] == "something" + # TODO: implement me def test_create_random_labels(self): pass From 6d6aad33dfd3b1fedfa9b7b0bc77b0fffc0ae3e9 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 15:01:42 -0400 Subject: [PATCH 53/54] yapf formatting --- tests/image/segmentation/test_serialization.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/image/segmentation/test_serialization.py b/tests/image/segmentation/test_serialization.py index 76fff0918a..a865223189 100644 --- a/tests/image/segmentation/test_serialization.py +++ b/tests/image/segmentation/test_serialization.py @@ -47,7 +47,9 @@ def test_serialize_fiftyone(self): sample = { DefaultDataKeys.PREDS: preds, - DefaultDataKeys.METADATA: {"filepath": "something"}, + DefaultDataKeys.METADATA: { + "filepath": "something" + }, } segmentation = serial.serialize(sample) From 691fc7f6ada59294b36a258fa6f2d3cf120d76b6 Mon Sep 17 00:00:00 2001 From: Eric Hofesmann Date: Tue, 15 Jun 2021 15:31:37 -0400 Subject: [PATCH 54/54] one more test... --- tests/image/detection/test_serialization.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/image/detection/test_serialization.py b/tests/image/detection/test_serialization.py index 9e1080a980..d4f7384786 100644 --- a/tests/image/detection/test_serialization.py +++ b/tests/image/detection/test_serialization.py @@ -14,8 +14,11 @@ def test_smoke(self): assert serial is not None def test_serialize_fiftyone(self): + labels = ['class_1', 'class_2', 'class_3'] serial = FiftyOneDetectionLabels() filepath_serial = FiftyOneDetectionLabels(return_filepath=True) + threshold_serial = FiftyOneDetectionLabels(threshold=0.9) + labels_serial = FiftyOneDetectionLabels(labels=labels) sample = { DefaultDataKeys.PREDS: [ @@ -35,8 +38,20 @@ def test_serialize_fiftyone(self): detections = serial.serialize(sample) assert len(detections.detections) == 1 assert detections.detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] + assert detections.detections[0].confidence == 0.5 + assert detections.detections[0].label == "0" detections = filepath_serial.serialize(sample) assert len(detections["predictions"].detections) == 1 assert detections["predictions"].detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] + assert detections["predictions"].detections[0].confidence == 0.5 assert detections["filepath"] == "something" + + detections = threshold_serial.serialize(sample) + assert len(detections.detections) == 0 + + detections = labels_serial.serialize(sample) + assert len(detections.detections) == 1 + assert detections.detections[0].bounding_box == [0.2, 0.3, 0.2, 0.2] + assert detections.detections[0].confidence == 0.5 + assert detections.detections[0].label == "class_1"