diff --git a/pkgs/development/python-modules/ome-zarr/default.nix b/pkgs/development/python-modules/ome-zarr/default.nix index 7da0d0cad4e8f..08d7a6857d4ad 100644 --- a/pkgs/development/python-modules/ome-zarr/default.nix +++ b/pkgs/development/python-modules/ome-zarr/default.nix @@ -1,9 +1,13 @@ { lib, buildPythonPackage, - pythonOlder, fetchFromGitHub, - pytestCheckHook, + + # build-system + setuptools, + setuptools-scm, + + # dependencies aiohttp, dask, distributed, @@ -11,9 +15,11 @@ numpy, requests, scikit-image, - setuptools, toolz, zarr, + + # tests + pytestCheckHook, }: buildPythonPackage rec { @@ -21,8 +27,6 @@ buildPythonPackage rec { version = "0.10.3"; pyproject = true; - disabled = pythonOlder "3.9"; - src = fetchFromGitHub { owner = "ome"; repo = "ome-zarr-py"; @@ -32,21 +36,24 @@ buildPythonPackage rec { build-system = [ setuptools + setuptools-scm ]; dependencies = [ - numpy + aiohttp dask distributed - zarr fsspec - aiohttp + numpy requests scikit-image toolz + zarr ] ++ fsspec.optional-dependencies.s3; - nativeCheckInputs = [ pytestCheckHook ]; + nativeCheckInputs = [ + pytestCheckHook + ]; disabledTests = [ # attempts to access network @@ -90,7 +97,7 @@ buildPythonPackage rec { meta = { description = "Implementation of next-generation file format (NGFF) specifications for storing bioimaging data in the cloud"; homepage = "https://pypi.org/project/ome-zarr"; - changelog = "https://github.com/ome/ome-zarr-py/blob/${src.tag}/CHANGELOG.md"; + changelog = "https://github.com/ome/ome-zarr-py/blob/v${version}/CHANGELOG.md"; license = lib.licenses.bsd2; maintainers = [ lib.maintainers.bcdarwin ]; mainProgram = "ome_zarr"; diff --git a/pkgs/development/python-modules/pyiceberg/default.nix b/pkgs/development/python-modules/pyiceberg/default.nix index ba91974b31011..1b6a75329b2cf 100644 --- a/pkgs/development/python-modules/pyiceberg/default.nix +++ b/pkgs/development/python-modules/pyiceberg/default.nix @@ -1,5 +1,6 @@ { lib, + stdenv, buildPythonPackage, fetchFromGitHub, @@ -49,7 +50,6 @@ pytest-mock, pytest-timeout, requests-mock, - pythonOlder, }: buildPythonPackage rec { @@ -185,61 +185,70 @@ buildPythonPackage rec { "tests/integration" ]; - disabledTests = [ - # botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL - "test_checking_if_a_file_exists" - "test_closing_a_file" - "test_fsspec_file_tell" - "test_fsspec_getting_length_of_file" - "test_fsspec_pickle_round_trip_s3" - "test_fsspec_raise_on_opening_file_not_found" - "test_fsspec_read_specified_bytes_for_file" - "test_fsspec_write_and_read_file" - "test_writing_avro_file" - - # Require unpackaged gcsfs - "test_fsspec_converting_an_outputfile_to_an_inputfile_gcs" - "test_fsspec_new_input_file_gcs" - "test_fsspec_new_output_file_gcs" - "test_fsspec_pickle_roundtrip_gcs" - - # Timeout (network access) - "test_fsspec_converting_an_outputfile_to_an_inputfile_adls" - "test_fsspec_new_abfss_output_file_adls" - "test_fsspec_new_input_file_adls" - "test_fsspec_pickle_round_trip_aldfs" - - # TypeError: pyarrow.lib.large_list() takes no keyword argument - # From tests/io/test_pyarrow_stats.py: - "test_bounds" - "test_column_metrics_mode" - "test_column_sizes" - "test_metrics_mode_counts" - "test_metrics_mode_full" - "test_metrics_mode_non_default_trunc" - "test_metrics_mode_none" - "test_null_and_nan_counts" - "test_offsets" - "test_read_missing_statistics" - "test_record_count" - "test_value_counts" - "test_write_and_read_stats_schema" - # From tests/io/test_pyarrow.py: - "test_list_type_to_pyarrow" - "test_projection_add_column" - "test_projection_list_of_structs" - "test_read_list" - "test_schema_compatible_missing_nullable_field_nested" - "test_schema_compatible_nested" - "test_schema_mismatch_missing_required_field_nested" - "test_schema_to_pyarrow_schema_exclude_field_ids" - "test_schema_to_pyarrow_schema_include_field_ids" - # From tests/io/test_pyarrow_visitor.py - "test_round_schema_conversion_nested" - - # Hangs forever (from tests/io/test_pyarrow.py) - "test_getting_length_of_file_gcs" - ]; + disabledTests = + [ + # botocore.exceptions.EndpointConnectionError: Could not connect to the endpoint URL + "test_checking_if_a_file_exists" + "test_closing_a_file" + "test_fsspec_file_tell" + "test_fsspec_getting_length_of_file" + "test_fsspec_pickle_round_trip_s3" + "test_fsspec_raise_on_opening_file_not_found" + "test_fsspec_read_specified_bytes_for_file" + "test_fsspec_write_and_read_file" + "test_writing_avro_file" + + # Require unpackaged gcsfs + "test_fsspec_converting_an_outputfile_to_an_inputfile_gcs" + "test_fsspec_new_input_file_gcs" + "test_fsspec_new_output_file_gcs" + "test_fsspec_pickle_roundtrip_gcs" + + # Timeout (network access) + "test_fsspec_converting_an_outputfile_to_an_inputfile_adls" + "test_fsspec_new_abfss_output_file_adls" + "test_fsspec_new_input_file_adls" + "test_fsspec_pickle_round_trip_aldfs" + + # TypeError: pyarrow.lib.large_list() takes no keyword argument + # From tests/io/test_pyarrow_stats.py: + "test_bounds" + "test_column_metrics_mode" + "test_column_sizes" + "test_metrics_mode_counts" + "test_metrics_mode_full" + "test_metrics_mode_non_default_trunc" + "test_metrics_mode_none" + "test_null_and_nan_counts" + "test_offsets" + "test_read_missing_statistics" + "test_record_count" + "test_value_counts" + "test_write_and_read_stats_schema" + # From tests/io/test_pyarrow.py: + "test_list_type_to_pyarrow" + "test_projection_add_column" + "test_projection_list_of_structs" + "test_read_list" + "test_schema_compatible_missing_nullable_field_nested" + "test_schema_compatible_nested" + "test_schema_mismatch_missing_required_field_nested" + "test_schema_to_pyarrow_schema_exclude_field_ids" + "test_schema_to_pyarrow_schema_include_field_ids" + # From tests/io/test_pyarrow_visitor.py + "test_round_schema_conversion_nested" + + # Hangs forever (from tests/io/test_pyarrow.py) + "test_getting_length_of_file_gcs" + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + # ImportError: The pyarrow installation is not built with support for 'GcsFileSystem' + "test_converting_an_outputfile_to_an_inputfile_gcs" + "test_new_input_file_gcs" + "test_new_output_file_gc" + ]; + + __darwinAllowLocalNetworking = true; meta = { description = "Python library for programmatic access to Apache Iceberg"; diff --git a/pkgs/development/python-modules/s3fs/default.nix b/pkgs/development/python-modules/s3fs/default.nix index 8586914416eba..af5ae376c84fe 100644 --- a/pkgs/development/python-modules/s3fs/default.nix +++ b/pkgs/development/python-modules/s3fs/default.nix @@ -1,35 +1,46 @@ { lib, - aiobotocore, - aiohttp, buildPythonPackage, + fetchFromGitHub, + + # buildInputs docutils, - fetchPypi, + + # build-system + setuptools, + + # dependencies + aiobotocore, + aiohttp, + fsspec, + + # tests flask, flask-cors, - fsspec, moto, pytestCheckHook, - pythonOlder, - setuptools, }: buildPythonPackage rec { pname = "s3fs"; - version = "2024.12.0"; + version = "2025.2.0"; pyproject = true; - disabled = pythonOlder "3.9"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-Gw86j1lGzKW6KYcdZ5KrHkUo7XYjJ9iu+vyBtzuZ/VY="; + src = fetchFromGitHub { + owner = "fsspec"; + repo = "s3fs"; + tag = version; + hash = "sha256-nnfvccORDspj54sRxL3d0hn4MpzKYGKE2Kl0v/wLaNw="; }; buildInputs = [ docutils ]; build-system = [ setuptools ]; + pythonRelaxDeps = [ + "fsspec" + ]; + dependencies = [ aiobotocore aiohttp @@ -48,12 +59,88 @@ buildPythonPackage rec { disabledTests = [ # require network access "test_async_close" + + # AssertionError: assert ['x', 'y'] == [] + "test_with_data" + + # AssertionError: assert ['1', 'x', 'y'] == [] + "test_clear_empty" + "test_no_dircache" + + # KeyError: 'ChecksumAlgorithm' + "test_info" + + # KeyError: + # del d[1] + "test_complex_keys" + + # TypeError: string indices must be integers, not 'str' + "test_bucket_versioning" + "test_bulk_delete" + "test_copy_with_source_and_destination_as_list" + "test_cp_directory_recursive" + "test_dynamic_add_rm" + "test_get_directory_to_existing_directory" + "test_get_directory_to_new_directory" + "test_get_directory_without_files_with_same_name_prefix" + "test_get_file_info_with_selector" + "test_get_file_to_existing_directory" + "test_get_file_to_file_in_existing_directory" + "test_get_file_to_file_in_new_directory" + "test_get_file_to_new_directory" + "test_get_glob_edge_cases" + "test_get_glob_to_existing_directory" + "test_get_glob_to_new_directory" + "test_get_list_of_files_to_existing_directory" + "test_get_list_of_files_to_new_directory" + "test_get_with_source_and_destination_as_list" + "test_move[False]" + "test_move[True]" + "test_new_bucket" + "test_new_bucket_auto" + "test_pipe_exclusive" + "test_put_directory_recursive" + "test_put_directory_to_existing_directory" + "test_put_directory_to_new_directory" + "test_put_directory_without_files_with_same_name_prefix" + "test_put_file_to_existing_directory" + "test_put_file_to_file_in_existing_directory" + "test_put_file_to_file_in_new_directory" + "test_put_file_to_new_directory" + "test_put_glob_edge_cases" + "test_put_glob_to_existing_directory" + "test_put_glob_to_new_directory" + "test_put_list_of_files_to_existing_directory" + "test_put_list_of_files_to_new_directory" + "test_rm" + "test_rm_invalidates_cache" + "test_rm_recursive_folder" + "test_s3_big_ls" + "test_s3fs_etag_preserving_multipart_copy" + "test_tags" + + # ExceptionGroup: errors while tearing down (2 sub-exceptions) + "test_copy_directory_to_existing_directory" + "test_copy_directory_to_new_directory" + "test_copy_directory_without_files_with_same_name_prefix" + "test_copy_file_to_existing_directory" + "test_copy_file_to_file_in_existing_directory" + "test_copy_file_to_file_in_new_directory" + "test_copy_file_to_new_directory" + "test_copy_glob_edge_cases" + "test_copy_glob_to_existing_directory" + "test_copy_glob_to_new_directory" + "test_copy_list_of_files_to_existing_directory" + "test_copy_list_of_files_to_new_directory" + "test_copy_two_files_new_directory" ]; + __darwinAllowLocalNetworking = true; + meta = { description = "Pythonic file interface for S3"; homepage = "https://github.com/fsspec/s3fs"; - changelog = "https://github.com/fsspec/s3fs/raw/${version}/docs/source/changelog.rst"; + changelog = "https://github.com/fsspec/s3fs/blob/${version}/docs/source/changelog.rst"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ teh ]; };