From ce1a78bede706900a12493c25a55b23742656b8e Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 18:49:01 +0100 Subject: [PATCH 01/12] Speed up graph tests --- .github/workflows/ci-testing.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 34eabb6cc3..9cb7f1f0a0 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -138,6 +138,12 @@ jobs: run: | pip install git+https://github.com/facebookresearch/vissl.git@master + - name: Install graph test dependencies + if: matrix.topic[0] == 'graph' + run: | + pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cpu.html + pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cpu.html + - name: Install dependencies run: | python --version From 323557e42d661a71aeb6c83da960df4c17c4620a Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 19:04:51 +0100 Subject: [PATCH 02/12] Move data set cache --- .github/workflows/ci-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 9cb7f1f0a0..1c50515b41 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -172,8 +172,8 @@ jobs: uses: actions/cache@v2 with: path: data # This path is specific to Ubuntu - key: lightning-flash-datasets-${{ hashFiles('tests/examples/test_scripts.py') }} - restore-keys: lightning-flash-datasets- + key: flash-datasets-${{ hashFiles('tests/examples/test_scripts.py') }} + restore-keys: flash-datasets- - name: Tests env: From 31ada981bfd8065b9374d2ae0c0ea8a01ee27aca Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 19:26:22 +0100 Subject: [PATCH 03/12] Debug --- .github/workflows/ci-testing.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 1c50515b41..95d19b77ed 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -175,6 +175,12 @@ jobs: key: flash-datasets-${{ hashFiles('tests/examples/test_scripts.py') }} restore-keys: flash-datasets- + - name: CUDA Version + run: | + import torch + print(torch.version.cuda) + shell: python + - name: Tests env: FLASH_TEST_TOPIC: ${{ join(matrix.topic,',') }} From ea9bae4c4e5e968ad21473411ba1789f914ed57f Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 19:30:20 +0100 Subject: [PATCH 04/12] Try fix --- .github/workflows/ci-testing.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 95d19b77ed..abd6101263 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -141,8 +141,8 @@ jobs: - name: Install graph test dependencies if: matrix.topic[0] == 'graph' run: | - pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cpu.html - pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cpu.html + pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cu102.html + pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cu102.html - name: Install dependencies run: | @@ -175,12 +175,6 @@ jobs: key: flash-datasets-${{ hashFiles('tests/examples/test_scripts.py') }} restore-keys: flash-datasets- - - name: CUDA Version - run: | - import torch - print(torch.version.cuda) - shell: python - - name: Tests env: FLASH_TEST_TOPIC: ${{ join(matrix.topic,',') }} From 889d08bc722f7839afcbd93c90c56881e398ae5b Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 22:06:31 +0100 Subject: [PATCH 05/12] Try fix --- .github/workflows/ci-testing.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index abd6101263..78667c6242 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -141,8 +141,9 @@ jobs: - name: Install graph test dependencies if: matrix.topic[0] == 'graph' run: | - pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cu102.html - pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cu102.html + pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cpu.html + pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cpu.html + pip install torch==1.9.0+cpu torchvision==0.10.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - name: Install dependencies run: | From 4eea22289b2dff787a7d0f880830266f2d98dc6e Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 22:11:14 +0100 Subject: [PATCH 06/12] Try fix --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 78667c6242..7b342f370a 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -141,9 +141,9 @@ jobs: - name: Install graph test dependencies if: matrix.topic[0] == 'graph' run: | + pip install torch==1.9.0+cpu torchvision==0.10.0+cpu -f https://download.pytorch.org/whl/torch_stable.html pip install torch-scatter -f https://data.pyg.org/whl/torch-1.9.0+cpu.html pip install torch-sparse -f https://data.pyg.org/whl/torch-1.9.0+cpu.html - pip install torch==1.9.0+cpu torchvision==0.10.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - name: Install dependencies run: | From 6b80c6a0b473ee22357f59f9fd3b72b0688cd445 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 22:38:49 +0100 Subject: [PATCH 07/12] Debug --- tests/examples/utils.py | 2 +- tests/graph/classification/test_data.py | 2 ++ tests/graph/classification/test_model.py | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/examples/utils.py b/tests/examples/utils.py index cf713fcbd1..0cbf9afec3 100644 --- a/tests/examples/utils.py +++ b/tests/examples/utils.py @@ -19,7 +19,7 @@ def call_script( filepath: str, args: Optional[List[str]] = None, - timeout: Optional[int] = 60 * 10, + timeout: Optional[int] = 60 * 5, ) -> Tuple[int, str, str]: with open(filepath) as original: data = original.read() diff --git a/tests/graph/classification/test_data.py b/tests/graph/classification/test_data.py index de4d08ff72..11d96f017e 100644 --- a/tests/graph/classification/test_data.py +++ b/tests/graph/classification/test_data.py @@ -41,6 +41,7 @@ def test_smoke(self): dm = GraphClassificationData() assert dm is not None + @pytest.skip() def test_from_datasets(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset @@ -80,6 +81,7 @@ def test_from_datasets(self, tmpdir): assert list(data.x.size())[1] == tudataset.num_features assert list(data.y.size()) == [2] + @pytest.skip() def test_transforms(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset diff --git a/tests/graph/classification/test_model.py b/tests/graph/classification/test_model.py index 0813a6fb3a..cb1a3242fa 100644 --- a/tests/graph/classification/test_model.py +++ b/tests/graph/classification/test_model.py @@ -36,6 +36,7 @@ def test_smoke(): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") +@pytest.skip() def test_train(tmpdir): """Tests that the model can be trained on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -47,6 +48,7 @@ def test_train(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") +@pytest.skip() def test_val(tmpdir): """Tests that the model can be validated on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -58,6 +60,7 @@ def test_val(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") +@pytest.skip() def test_test(tmpdir): """Tests that the model can be tested on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -69,6 +72,7 @@ def test_test(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") +@pytest.skip() def test_predict_dataset(tmpdir): """Tests that we can generate predictions from a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -79,6 +83,7 @@ def test_predict_dataset(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") +@pytest.skip() def test_cli(): cli_args = ["flash", "graph_classification", "--trainer.fast_dev_run", "True"] with mock.patch("sys.argv", cli_args): From 57df873f88980c110d9a70be99194e0478141bfe Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 22:46:18 +0100 Subject: [PATCH 08/12] Debug --- tests/graph/classification/test_data.py | 4 ++-- tests/graph/classification/test_model.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/graph/classification/test_data.py b/tests/graph/classification/test_data.py index 11d96f017e..d1a6a6c810 100644 --- a/tests/graph/classification/test_data.py +++ b/tests/graph/classification/test_data.py @@ -41,7 +41,7 @@ def test_smoke(self): dm = GraphClassificationData() assert dm is not None - @pytest.skip() + @pytest.mark.skip() def test_from_datasets(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset @@ -81,7 +81,7 @@ def test_from_datasets(self, tmpdir): assert list(data.x.size())[1] == tudataset.num_features assert list(data.y.size()) == [2] - @pytest.skip() + @pytest.mark.skip() def test_transforms(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset diff --git a/tests/graph/classification/test_model.py b/tests/graph/classification/test_model.py index cb1a3242fa..70a7d8db7a 100644 --- a/tests/graph/classification/test_model.py +++ b/tests/graph/classification/test_model.py @@ -36,7 +36,7 @@ def test_smoke(): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.skip() +@pytest.mark.skip() def test_train(tmpdir): """Tests that the model can be trained on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -48,7 +48,7 @@ def test_train(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.skip() +@pytest.mark.skip() def test_val(tmpdir): """Tests that the model can be validated on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -60,7 +60,7 @@ def test_val(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.skip() +@pytest.mark.skip() def test_test(tmpdir): """Tests that the model can be tested on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -72,7 +72,7 @@ def test_test(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.skip() +@pytest.mark.skip() def test_predict_dataset(tmpdir): """Tests that we can generate predictions from a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -83,7 +83,7 @@ def test_predict_dataset(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.skip() +@pytest.mark.skip() def test_cli(): cli_args = ["flash", "graph_classification", "--trainer.fast_dev_run", "True"] with mock.patch("sys.argv", cli_args): From 652256b714311eebddee39d006304fe7d1d11b02 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 23:12:08 +0100 Subject: [PATCH 09/12] Remove auto num workers --- flash/core/data/data_module.py | 10 +--------- tests/examples/utils.py | 2 +- tests/graph/classification/test_data.py | 2 -- tests/graph/classification/test_model.py | 5 ----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index ef5e118cc3..093d014f9c 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import platform from typing import ( Any, Callable, @@ -98,7 +97,7 @@ def __init__( data_fetcher: Optional[BaseDataFetcher] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: Optional[int] = 0, sampler: Optional[Type[Sampler]] = None, ) -> None: @@ -137,13 +136,6 @@ def __init__( self.predict_dataloader = self._predict_dataloader self.batch_size = batch_size - - # TODO: figure out best solution for setting num_workers - if num_workers is None: - if platform.system() in ("Darwin", "Windows"): - num_workers = 0 - else: - num_workers = os.cpu_count() self.num_workers = num_workers self.sampler = sampler diff --git a/tests/examples/utils.py b/tests/examples/utils.py index 0cbf9afec3..cf713fcbd1 100644 --- a/tests/examples/utils.py +++ b/tests/examples/utils.py @@ -19,7 +19,7 @@ def call_script( filepath: str, args: Optional[List[str]] = None, - timeout: Optional[int] = 60 * 5, + timeout: Optional[int] = 60 * 10, ) -> Tuple[int, str, str]: with open(filepath) as original: data = original.read() diff --git a/tests/graph/classification/test_data.py b/tests/graph/classification/test_data.py index d1a6a6c810..de4d08ff72 100644 --- a/tests/graph/classification/test_data.py +++ b/tests/graph/classification/test_data.py @@ -41,7 +41,6 @@ def test_smoke(self): dm = GraphClassificationData() assert dm is not None - @pytest.mark.skip() def test_from_datasets(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset @@ -81,7 +80,6 @@ def test_from_datasets(self, tmpdir): assert list(data.x.size())[1] == tudataset.num_features assert list(data.y.size()) == [2] - @pytest.mark.skip() def test_transforms(self, tmpdir): tudataset = TUDataset(root=tmpdir, name="KKI") train_dataset = tudataset diff --git a/tests/graph/classification/test_model.py b/tests/graph/classification/test_model.py index 70a7d8db7a..0813a6fb3a 100644 --- a/tests/graph/classification/test_model.py +++ b/tests/graph/classification/test_model.py @@ -36,7 +36,6 @@ def test_smoke(): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.mark.skip() def test_train(tmpdir): """Tests that the model can be trained on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -48,7 +47,6 @@ def test_train(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.mark.skip() def test_val(tmpdir): """Tests that the model can be validated on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -60,7 +58,6 @@ def test_val(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.mark.skip() def test_test(tmpdir): """Tests that the model can be tested on a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -72,7 +69,6 @@ def test_test(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.mark.skip() def test_predict_dataset(tmpdir): """Tests that we can generate predictions from a pytorch geometric dataset.""" tudataset = datasets.TUDataset(root=tmpdir, name="KKI") @@ -83,7 +79,6 @@ def test_predict_dataset(tmpdir): @pytest.mark.skipif(not _GRAPH_TESTING, reason="pytorch geometric isn't installed") -@pytest.mark.skip() def test_cli(): cli_args = ["flash", "graph_classification", "--trainer.fast_dev_run", "True"] with mock.patch("sys.argv", cli_args): From 320f073b9aefa60a91b339a14f35da8189e9c1e9 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 23:24:10 +0100 Subject: [PATCH 10/12] Update default num_workers --- flash/audio/classification/cli.py | 3 +-- flash/audio/speech_recognition/cli.py | 3 +-- flash/core/data/data_module.py | 24 +++++++++++++---------- flash/core/utilities/imports.py | 2 +- flash/graph/classification/cli.py | 3 +-- flash/image/classification/cli.py | 5 ++--- flash/image/classification/data.py | 4 ++-- flash/image/detection/cli.py | 3 +-- flash/image/detection/data.py | 6 +++--- flash/image/instance_segmentation/cli.py | 2 +- flash/image/instance_segmentation/data.py | 4 ++-- flash/image/keypoint_detection/cli.py | 2 +- flash/image/keypoint_detection/data.py | 2 +- flash/image/segmentation/cli.py | 3 +-- flash/image/segmentation/data.py | 4 ++-- flash/image/style_transfer/cli.py | 3 +-- flash/pointcloud/detection/cli.py | 3 +-- flash/pointcloud/detection/data.py | 2 +- flash/pointcloud/segmentation/cli.py | 3 +-- flash/tabular/classification/cli.py | 3 +-- flash/tabular/data.py | 4 ++-- flash/template/classification/data.py | 2 +- flash/text/classification/cli.py | 5 ++--- flash/text/question_answering/cli.py | 3 +-- flash/text/question_answering/data.py | 6 +++--- flash/text/seq2seq/summarization/cli.py | 3 +-- flash/text/seq2seq/translation/cli.py | 3 +-- flash/video/classification/cli.py | 3 +-- tests/core/test_data.py | 7 +------ 29 files changed, 52 insertions(+), 68 deletions(-) diff --git a/flash/audio/classification/cli.py b/flash/audio/classification/cli.py index c198a99239..c69b1e540c 100644 --- a/flash/audio/classification/cli.py +++ b/flash/audio/classification/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.audio import AudioClassificationData from flash.core.data.utils import download_data @@ -23,7 +22,7 @@ def from_urban8k( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> AudioClassificationData: """Downloads and loads the Urban 8k sounds images data set.""" diff --git a/flash/audio/speech_recognition/cli.py b/flash/audio/speech_recognition/cli.py index 9bbdb48df8..f8a7ad26dd 100644 --- a/flash/audio/speech_recognition/cli.py +++ b/flash/audio/speech_recognition/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.audio import SpeechRecognition, SpeechRecognitionData from flash.core.data.utils import download_data @@ -23,7 +22,7 @@ def from_timit( val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> SpeechRecognitionData: """Downloads and loads the timit data set.""" diff --git a/flash/core/data/data_module.py b/flash/core/data/data_module.py index 093d014f9c..84f07734f1 100644 --- a/flash/core/data/data_module.py +++ b/flash/core/data/data_module.py @@ -97,7 +97,7 @@ def __init__( data_fetcher: Optional[BaseDataFetcher] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = 0, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, ) -> None: @@ -136,7 +136,11 @@ def __init__( self.predict_dataloader = self._predict_dataloader self.batch_size = batch_size + + if num_workers is None: + num_workers = 0 self.num_workers = num_workers + self.sampler = sampler self.set_running_stages() @@ -460,7 +464,7 @@ def from_data_source( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -561,7 +565,7 @@ def from_folders( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -635,7 +639,7 @@ def from_files( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -712,7 +716,7 @@ def from_tensors( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -799,7 +803,7 @@ def from_numpy( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -885,7 +889,7 @@ def from_json( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, field: Optional[str] = None, **preprocess_kwargs: Any, @@ -995,7 +999,7 @@ def from_csv( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -1079,7 +1083,7 @@ def from_datasets( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -1160,7 +1164,7 @@ def from_fiftyone( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ) -> "DataModule": """Creates a :class:`~flash.core.data.data_module.DataModule` object diff --git a/flash/core/utilities/imports.py b/flash/core/utilities/imports.py index d3eff5a4d2..73524ee7a0 100644 --- a/flash/core/utilities/imports.py +++ b/flash/core/utilities/imports.py @@ -103,7 +103,7 @@ def _compare_version(package: str, op, version) -> bool: _ALBUMENTATIONS_AVAILABLE = _module_available("albumentations") if _PIL_AVAILABLE: - from PIL import Image + pass else: class MetaImage(type): diff --git a/flash/graph/classification/cli.py b/flash/graph/classification/cli.py index f79af259d8..d8fd18702c 100644 --- a/flash/graph/classification/cli.py +++ b/flash/graph/classification/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.utilities.flash_cli import FlashCLI from flash.graph import GraphClassificationData, GraphClassifier @@ -23,7 +22,7 @@ def from_tu_dataset( name: str = "KKI", val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> GraphClassificationData: """Downloads and loads the TU Dataset.""" diff --git a/flash/image/classification/cli.py b/flash/image/classification/cli.py index 6804c909f8..4056387b86 100644 --- a/flash/image/classification/cli.py +++ b/flash/image/classification/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -22,7 +21,7 @@ def from_hymenoptera( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> ImageClassificationData: """Downloads and loads the Hymenoptera (Ants, Bees) data set.""" @@ -38,7 +37,7 @@ def from_hymenoptera( def from_movie_posters( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> ImageClassificationData: """Downloads and loads the movie posters genre classification data set.""" diff --git a/flash/image/classification/data.py b/flash/image/classification/data.py index 0cf7bc2ca8..5f035949c2 100644 --- a/flash/image/classification/data.py +++ b/flash/image/classification/data.py @@ -128,7 +128,7 @@ def from_data_frame( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": @@ -225,7 +225,7 @@ def from_csv( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, **preprocess_kwargs: Any, ) -> "DataModule": diff --git a/flash/image/detection/cli.py b/flash/image/detection/cli.py index 8c2eb0c3d1..f955e34bbe 100644 --- a/flash/image/detection/cli.py +++ b/flash/image/detection/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -23,7 +22,7 @@ def from_coco_128( val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> ObjectDetectionData: """Downloads and loads the COCO 128 data set.""" diff --git a/flash/image/detection/data.py b/flash/image/detection/data.py index 9b00375d99..9a7e5c31fa 100644 --- a/flash/image/detection/data.py +++ b/flash/image/detection/data.py @@ -200,7 +200,7 @@ def from_coco( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.detection.data.ObjectDetectionData` object from the given data folders @@ -279,7 +279,7 @@ def from_voc( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.detection.data.ObjectDetectionData` object from the given data folders @@ -358,7 +358,7 @@ def from_via( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.detection.data.ObjectDetectionData` object from the given data folders diff --git a/flash/image/instance_segmentation/cli.py b/flash/image/instance_segmentation/cli.py index 63b2538d09..97960ae5c9 100644 --- a/flash/image/instance_segmentation/cli.py +++ b/flash/image/instance_segmentation/cli.py @@ -28,7 +28,7 @@ def from_pets( val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, parser: Optional[Callable] = None, **preprocess_kwargs, ) -> InstanceSegmentationData: diff --git a/flash/image/instance_segmentation/data.py b/flash/image/instance_segmentation/data.py index 91a1e8eeb1..ab68c3d5ff 100644 --- a/flash/image/instance_segmentation/data.py +++ b/flash/image/instance_segmentation/data.py @@ -92,7 +92,7 @@ def from_coco( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.instance_segmentation.data.InstanceSegmentationData` object from the @@ -171,7 +171,7 @@ def from_voc( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.instance_segmentation.data.InstanceSegmentationData` object from the diff --git a/flash/image/keypoint_detection/cli.py b/flash/image/keypoint_detection/cli.py index 5f8d0feade..959328a51c 100644 --- a/flash/image/keypoint_detection/cli.py +++ b/flash/image/keypoint_detection/cli.py @@ -27,7 +27,7 @@ def from_biwi( val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, parser: Optional[Callable] = None, **preprocess_kwargs, ) -> KeypointDetectionData: diff --git a/flash/image/keypoint_detection/data.py b/flash/image/keypoint_detection/data.py index 0e7f700e4a..97948a7d40 100644 --- a/flash/image/keypoint_detection/data.py +++ b/flash/image/keypoint_detection/data.py @@ -90,7 +90,7 @@ def from_coco( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.image.keypoint_detection.data.KeypointDetectionData` object from the given data diff --git a/flash/image/segmentation/cli.py b/flash/image/segmentation/cli.py index 64cb0c3d93..2e92877015 100644 --- a/flash/image/segmentation/cli.py +++ b/flash/image/segmentation/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -24,7 +23,7 @@ def from_carla( num_classes: int = 21, val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> SemanticSegmentationData: """Downloads and loads the CARLA capture data set.""" diff --git a/flash/image/segmentation/data.py b/flash/image/segmentation/data.py index 42787df9bc..d9d2a69e57 100644 --- a/flash/image/segmentation/data.py +++ b/flash/image/segmentation/data.py @@ -318,7 +318,7 @@ def from_data_source( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ) -> "DataModule": @@ -376,7 +376,7 @@ def from_folders( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, num_classes: Optional[int] = None, labels_map: Dict[int, Tuple[int, int, int]] = None, **preprocess_kwargs, diff --git a/flash/image/style_transfer/cli.py b/flash/image/style_transfer/cli.py index 0fec347021..0aab00a4e3 100644 --- a/flash/image/style_transfer/cli.py +++ b/flash/image/style_transfer/cli.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Optional import flash from flash.core.data.utils import download_data @@ -24,7 +23,7 @@ def from_coco_128( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> StyleTransferData: """Downloads and loads the COCO 128 data set.""" diff --git a/flash/pointcloud/detection/cli.py b/flash/pointcloud/detection/cli.py index 01a4c329ce..1acbef5efa 100644 --- a/flash/pointcloud/detection/cli.py +++ b/flash/pointcloud/detection/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -22,7 +21,7 @@ def from_kitti( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> PointCloudObjectDetectorData: """Downloads and loads the KITTI data set.""" diff --git a/flash/pointcloud/detection/data.py b/flash/pointcloud/detection/data.py index 411d3a498f..e565c358b7 100644 --- a/flash/pointcloud/detection/data.py +++ b/flash/pointcloud/detection/data.py @@ -89,7 +89,7 @@ def from_folders( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Type[Sampler]] = None, scans_folder_name: Optional[str] = "scans", labels_folder_name: Optional[str] = "labels", diff --git a/flash/pointcloud/segmentation/cli.py b/flash/pointcloud/segmentation/cli.py index 57d1125f9b..26a147d68b 100644 --- a/flash/pointcloud/segmentation/cli.py +++ b/flash/pointcloud/segmentation/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -22,7 +21,7 @@ def from_kitti( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> PointCloudSegmentationData: """Downloads and loads the semantic KITTI data set.""" diff --git a/flash/tabular/classification/cli.py b/flash/tabular/classification/cli.py index 63eff2458f..6787b1c8d6 100644 --- a/flash/tabular/classification/cli.py +++ b/flash/tabular/classification/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -22,7 +21,7 @@ def from_titanic( batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> TabularClassificationData: """Downloads and loads the Titanic data set.""" diff --git a/flash/tabular/data.py b/flash/tabular/data.py index da36d726ce..6a6f87fa5a 100644 --- a/flash/tabular/data.py +++ b/flash/tabular/data.py @@ -343,7 +343,7 @@ def from_data_frame( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.tabular.data.TabularData` object from the given data frames. @@ -450,7 +450,7 @@ def from_csv( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ) -> "DataModule": """Creates a :class:`~flash.tabular.data.TabularData` object from the given CSV files. diff --git a/flash/template/classification/data.py b/flash/template/classification/data.py index f81111bc3c..fc311c0a6a 100644 --- a/flash/template/classification/data.py +++ b/flash/template/classification/data.py @@ -176,7 +176,7 @@ def from_sklearn( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """This is our custom ``from_*`` method. It expects scikit-learn ``Bunch`` objects as input and passes them diff --git a/flash/text/classification/cli.py b/flash/text/classification/cli.py index 42499bb53f..0b7be2bd11 100644 --- a/flash/text/classification/cli.py +++ b/flash/text/classification/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -23,7 +22,7 @@ def from_imdb( backbone: str = "prajjwal1/bert-medium", batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> TextClassificationData: """Downloads and loads the IMDB sentiment classification data set.""" @@ -44,7 +43,7 @@ def from_toxic( backbone: str = "unitary/toxic-bert", val_split: float = 0.1, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> TextClassificationData: """Downloads and loads the Jigsaw toxic comments data set.""" diff --git a/flash/text/question_answering/cli.py b/flash/text/question_answering/cli.py index 12932ae930..0433a068c8 100644 --- a/flash/text/question_answering/cli.py +++ b/flash/text/question_answering/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -23,7 +22,7 @@ def from_squad( backbone: str = "distilbert-base-uncased", batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> QuestionAnsweringData: """Downloads and loads the XSum data set.""" diff --git a/flash/text/question_answering/data.py b/flash/text/question_answering/data.py index 1948ca9b21..35f3af8df7 100644 --- a/flash/text/question_answering/data.py +++ b/flash/text/question_answering/data.py @@ -645,7 +645,7 @@ def from_squad_v2( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs: Any, ): """Creates a :class:`~flash.text.question_answering.data.QuestionAnsweringData` object from the given data @@ -713,7 +713,7 @@ def from_json( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Sampler] = None, field: Optional[str] = None, **preprocess_kwargs: Any, @@ -814,7 +814,7 @@ def from_csv( preprocess: Optional[Preprocess] = None, val_split: Optional[float] = None, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, sampler: Optional[Sampler] = None, **preprocess_kwargs: Any, ) -> "DataModule": diff --git a/flash/text/seq2seq/summarization/cli.py b/flash/text/seq2seq/summarization/cli.py index 666dd87f40..25003cb58b 100644 --- a/flash/text/seq2seq/summarization/cli.py +++ b/flash/text/seq2seq/summarization/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -23,7 +22,7 @@ def from_xsum( backbone: str = "sshleifer/distilbart-xsum-1-1", batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> SummarizationData: """Downloads and loads the XSum data set.""" diff --git a/flash/text/seq2seq/translation/cli.py b/flash/text/seq2seq/translation/cli.py index 1609cb4de0..66ec698791 100644 --- a/flash/text/seq2seq/translation/cli.py +++ b/flash/text/seq2seq/translation/cli.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -23,7 +22,7 @@ def from_wmt_en_ro( backbone: str = "Helsinki-NLP/opus-mt-en-ro", batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> TranslationData: """Downloads and loads the WMT EN RO data set.""" diff --git a/flash/video/classification/cli.py b/flash/video/classification/cli.py index 840386506b..3053d0c1ca 100644 --- a/flash/video/classification/cli.py +++ b/flash/video/classification/cli.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from typing import Optional from flash.core.data.utils import download_data from flash.core.utilities.flash_cli import FlashCLI @@ -26,7 +25,7 @@ def from_kinetics( clip_duration: int = 1, decode_audio: bool = False, batch_size: int = 4, - num_workers: Optional[int] = None, + num_workers: int = 0, **preprocess_kwargs, ) -> VideoClassificationData: """Downloads and loads the Kinetics data set.""" diff --git a/tests/core/test_data.py b/tests/core/test_data.py index 156669a657..7c77f69075 100644 --- a/tests/core/test_data.py +++ b/tests/core/test_data.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import platform - import torch from flash import DataModule @@ -54,7 +52,4 @@ def test_dataloaders(): def test_cpu_count_none(): train_ds = DummyDataset() dm = DataModule(train_ds, num_workers=None) - if platform.system() == "Darwin" or platform.system() == "Windows": - assert dm.num_workers == 0 - else: - assert dm.num_workers > 0 + assert dm.num_workers == 0 From bbe4f2cef8a4a6dac1fe1a2c1cc8d2e51f43e5c0 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 23:28:14 +0100 Subject: [PATCH 11/12] Fixes --- flash/core/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flash/core/utilities/imports.py b/flash/core/utilities/imports.py index 73524ee7a0..8ca24ab266 100644 --- a/flash/core/utilities/imports.py +++ b/flash/core/utilities/imports.py @@ -103,7 +103,7 @@ def _compare_version(package: str, op, version) -> bool: _ALBUMENTATIONS_AVAILABLE = _module_available("albumentations") if _PIL_AVAILABLE: - pass + from PIL import Image # noqa: F401 else: class MetaImage(type): From 6e8e614c84e0ce9cb29db0a3f1f095cb5be9ed57 Mon Sep 17 00:00:00 2001 From: Ethan Harris Date: Tue, 14 Sep 2021 23:32:33 +0100 Subject: [PATCH 12/12] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ed31f7c3..8cda700e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Changed +- Changed the default `num_workers` on linux to `0` (matching the default for other OS) ([#759](https://github.com/PyTorchLightning/lightning-flash/pull/759)) + ### Fixed ## [0.5.0] - 2021-09-07