diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 809907a3d1c..616232d202c 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -65,6 +65,7 @@ if "%JOB%" NEQ "Build_Debug" ( "ninja" ^ "nomkl" ^ "pandas" ^ + "fsspec" ^ "python=%PYTHON%" ^ || exit /B ) diff --git a/python/pyarrow/filesystem.py b/python/pyarrow/filesystem.py index c634984a70b..d6c11f5e8dd 100644 --- a/python/pyarrow/filesystem.py +++ b/python/pyarrow/filesystem.py @@ -446,10 +446,6 @@ def _ensure_filesystem(fs): for mro in inspect.getmro(fs_type): if mro.__name__ == 'S3FileSystem': return S3FSWrapper(fs) - # In case its a simple LocalFileSystem (e.g. dask) use native arrow - # FS - elif mro.__name__ == 'LocalFileSystem': - return LocalFileSystem._get_instance() if "fsspec" in sys.modules: fsspec = sys.modules["fsspec"] diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 37e104303af..8b4ec900963 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -4339,3 +4339,19 @@ def test_parquet_dataset_new_filesystem(tempdir): dataset = pq.ParquetDataset('.', filesystem=filesystem) result = dataset.read() assert result.equals(table) + + +def test_parquet_dataset_partitions_piece_path_with_fsspec(tempdir): + # ARROW-10462 ensure that on Windows we properly use posix-style paths + # as used by fsspec + fsspec = pytest.importorskip("fsspec") + filesystem = fsspec.filesystem('file') + table = pa.table({'a': [1, 2, 3]}) + pq.write_table(table, tempdir / 'data.parquet') + + # pass a posix-style path (using "/" also on Windows) + path = str(tempdir).replace("\\", "/") + dataset = pq.ParquetDataset(path, filesystem=filesystem) + # ensure the piece path is also posix-style + expected = path + "/data.parquet" + assert dataset.pieces[0].path == expected