python3Packages: update boto and fsspec ecosystem

pkgs/by-name/aw/awscli/package.nix

@@ -1,7 +1,7 @@
 {
   lib,
-  python3,
-  fetchPypi,
+  python3Packages,
+  fetchFromGitHub,
   groff,
   less,
   nix-update-script,
@@ -10,36 +10,38 @@
 }:
 
 let
-  self = python3.pkgs.buildPythonApplication rec {
+  self = python3Packages.buildPythonApplication rec {
     pname = "awscli";
     # N.B: if you change this, change botocore and boto3 to a matching version too
     # check e.g. https://github.com/aws/aws-cli/blob/1.33.21/setup.py
-    version = "1.36.40";
+    version = "1.37.21";
     pyproject = true;
 
-    src = fetchPypi {
-      inherit pname version;
-      hash = "sha256-4qiPiNwW1cDyY3mv1vJUCX5T6LNMghZOWfQWXbbubfo=";
+    src = fetchFromGitHub {
+      owner = "aws";
+      repo = "aws-cli";
+      tag = version;
+      hash = "sha256-gKRWhOhZjGhPVIG6KgCyDqxuyBGbaS8bHD7vnJ4gA+o=";
     };
 
     pythonRelaxDeps = [
       # botocore must not be relaxed
-      "colorama"
       "docutils"
       "rsa"
     ];
 
-    build-system = [
-      python3.pkgs.setuptools
+    build-system = with python3Packages; [
+      setuptools
     ];
 
-    dependencies = with python3.pkgs; [
+    dependencies = with python3Packages; [
       botocore
+      docutils
       s3transfer
+      pyyaml
       colorama
-      docutils
       rsa
-      pyyaml
+
       groff
       less
     ];
@@ -59,14 +61,14 @@ let
     installCheckPhase = ''
       runHook preInstallCheck
 
-      $out/bin/aws --version | grep "${python3.pkgs.botocore.version}"
+      $out/bin/aws --version | grep "${python3Packages.botocore.version}"
       $out/bin/aws --version | grep "${version}"
 
       runHook postInstallCheck
     '';
 
     passthru = {
-      python = python3; # for aws_shell
+      python = python3Packages.python; # for aws_shell
       updateScript = nix-update-script {
         extraArgs = [ "--version=skip" ];
       };

pkgs/development/python-modules/aioboto3/default.nix

@@ -13,12 +13,11 @@
   pytest-asyncio,
   pytestCheckHook,
   pythonOlder,
-  requests,
 }:
 
 buildPythonPackage rec {
   pname = "aioboto3";
-  version = "13.1.1";
+  version = "13.4.0";
   pyproject = true;
 
   disabled = pythonOlder "3.8";
@@ -27,16 +26,18 @@ buildPythonPackage rec {
     owner = "terrycain";
     repo = "aioboto3";
     tag = "v${version}";
-    hash = "sha256-g86RKQxTcfG1CIH3gfgn9Vl9JxUkeC1ztmLk4q/MVn0=";
+    hash = "sha256-o3PynPW6nPvbBrsw+HU2fJheVRpCHCb0EnJdmseorsE=";
   };
 
+  pythonRelaxDeps = [
+    "aiobotocore"
+  ];
+
   build-system = [
     poetry-core
     poetry-dynamic-versioning
   ];
 
-  pythonRelaxDeps = [ "aiobotocore" ];
-
   dependencies = [
     aiobotocore
     aiofiles
@@ -47,31 +48,23 @@ buildPythonPackage rec {
     s3cse = [ cryptography ];
   };
 
-  nativeCheckInputs = [
-    dill
-    moto
-    pytest-asyncio
-    pytestCheckHook
-    requests
-  ] ++ lib.flatten (builtins.attrValues optional-dependencies);
+  nativeCheckInputs =
+    [
+      dill
+      moto
+      pytest-asyncio
+      pytestCheckHook
+    ]
+    ++ moto.optional-dependencies.server
+    ++ lib.flatten (builtins.attrValues optional-dependencies);
 
   pythonImportsCheck = [ "aioboto3" ];
 
-  disabledTests = [
-    # Our moto package is not ready to support more tests
-    "encrypt_decrypt_aes_cbc"
-    "test_chalice_async"
-    "test_dynamo"
-    "test_flush_doesnt_reset_item_buffer"
-    "test_kms"
-    "test_s3"
-  ];
-
-  meta = with lib; {
+  meta = {
     description = "Wrapper to use boto3 resources with the aiobotocore async backend";
     homepage = "https://github.com/terrycain/aioboto3";
     changelog = "https://github.com/terrycain/aioboto3/blob/${src.rev}/CHANGELOG.rst";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ mbalatsko ];
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ mbalatsko ];
   };
 }

pkgs/development/python-modules/aiobotocore/default.nix

@@ -1,27 +1,30 @@
 {
   lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  pythonOlder,
   aiohttp,
   aioitertools,
   botocore,
-  buildPythonPackage,
+  python-dateutil,
+  jmespath,
+  multidict,
+  urllib3,
+  wrapt,
   dill,
-  fetchFromGitHub,
-  flask,
-  flask-cors,
-  awscli,
   moto,
+  pytest-asyncio,
+  time-machine,
+  werkzeug,
+  awscli,
   boto3,
   setuptools,
-  pytest-asyncio,
   pytestCheckHook,
-  pythonAtLeast,
-  pythonOlder,
-  wrapt,
 }:
 
 buildPythonPackage rec {
   pname = "aiobotocore";
-  version = "2.18.0";
+  version = "2.19.0";
   pyproject = true;
 
   disabled = pythonOlder "3.8";
@@ -30,19 +33,25 @@ buildPythonPackage rec {
     owner = "aio-libs";
     repo = "aiobotocore";
     tag = version;
-    hash = "sha256-4S0CZtk5k+xke2WVwYrNyvP2up32xrOcdKNP72kO6zQ=";
+    hash = "sha256-8wtWIkGja4zb2OoYALH9hTR6i90sIjIjYWTUulfYIYA=";
   };
 
   # Relax version constraints: aiobotocore works with newer botocore versions
   # the pinning used to match some `extras_require` we're not using.
   pythonRelaxDeps = [ "botocore" ];
 
-  build-system = [ setuptools ];
+  build-system = [
+    setuptools
+  ];
 
   dependencies = [
     aiohttp
     aioitertools
     botocore
+    python-dateutil
+    jmespath
+    multidict
+    urllib3
     wrapt
   ];
 
@@ -53,57 +62,40 @@ buildPythonPackage rec {
 
   nativeCheckInputs = [
     dill
-    flask
-    flask-cors
     moto
     pytest-asyncio
+    time-machine
+    werkzeug
     pytestCheckHook
-  ];
+  ] ++ moto.optional-dependencies.server;
 
   pythonImportsCheck = [ "aiobotocore" ];
 
   disabledTestPaths = [
-    # Tests require network access
-    "tests/python3.8/"
+    # Test requires network access
+    "tests/test_version.py"
+    # Test not compatible with latest moto
+    "tests/boto_tests/unit/test_eventstream.py"
+    "tests/python3.8/test_eventstreams.py"
     "tests/test_basic_s3.py"
     "tests/test_batch.py"
     "tests/test_dynamodb.py"
     "tests/test_ec2.py"
-    "tests/test_eventstreams.py"
     "tests/test_lambda.py"
     "tests/test_monitor.py"
-    "tests/test_mturk.py"
     "tests/test_patches.py"
     "tests/test_sns.py"
     "tests/test_sqs.py"
-    "tests/test_version.py"
     "tests/test_waiter.py"
-    # moto compat issues
-    "tests/boto_tests/unit/test_signers.py"
   ];
 
-  disabledTests =
-    [
-      "test_get_credential"
-      "test_load_sso_credentials_without_cache"
-      "test_load_sso_credentials"
-      "test_required_config_not_set"
-      "test_sso_cred_fetcher_raises_helpful_message_on_unauthorized_exception"
-      "test_sso_credential_fetcher_can_fetch_credentials"
-    ]
-    ++ lib.optionals (pythonAtLeast "3.12.") [
-      # AttributeError: 'called_with' is not a valid assertion. Use a spec for the mock if 'called_with' is meant to be an attribute.
-      "test_max_rate_updated_on_success_response"
-      "test_max_rate_cant_exceed_20_percent_max"
-    ];
-
   __darwinAllowLocalNetworking = true;
 
-  meta = with lib; {
+  meta = {
     description = "Python client for amazon services";
     homepage = "https://github.com/aio-libs/aiobotocore";
     changelog = "https://github.com/aio-libs/aiobotocore/releases/tag/${src.tag}";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ teh ];
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ teh ];
   };
 }

pkgs/development/python-modules/boto3/default.nix

@@ -25,12 +25,12 @@ buildPythonPackage rec {
     owner = "boto";
     repo = "boto3";
     tag = version;
-    hash = "sha256-b08tC8EA6iW0O/7rseD9pTkKh/cJ2fe3xJZkEqxS6VI=";
+    hash = "sha256-89GUr0isFEKmBevWgPW5z4uU1zOTQ1kM8RX1mlsvdXw=";
   };
 
-  build-system = [ setuptools ];
-
-  pythonRelaxDeps = [ "s3transfer" ];
+  build-system = [
+    setuptools
+  ];
 
   dependencies = [
     botocore
@@ -51,7 +51,7 @@ buildPythonPackage rec {
   ];
 
   optional-dependencies = {
-    crt = [ botocore.optional-dependencies.crt ];
+    crt = botocore.optional-dependencies.crt;
   };
 
   meta = {

pkgs/development/python-modules/botocore/default.nix

@@ -2,7 +2,7 @@
   lib,
   awscrt,
   buildPythonPackage,
-  fetchPypi,
+  fetchFromGitHub,
 
   # build-system
   setuptools,
@@ -19,17 +19,19 @@
 
 buildPythonPackage rec {
   pname = "botocore";
-  version = "1.35.99"; # N.B: if you change this, change boto3 and awscli to a matching version
+  version = "1.36.21"; # N.B: if you change this, change boto3 and awscli to a matching version
   pyproject = true;
 
-  src = fetchPypi {
-    inherit pname version;
-    hash = "sha256-HqtE6WnDnF89mjEEoINsJHFVeaRV8Ss5eaMdfN5Rs8M=";
+  src = fetchFromGitHub {
+    owner = "boto";
+    repo = "botocore";
+    tag = version;
+    hash = "sha256-wk3KCRagEju4ywJfoBR8/4dH3xYgzGgaSHavDYCd5XY=";
   };
 
-  pythonRelaxDeps = [ "urllib3" ];
-
-  build-system = [ setuptools ];
+  build-system = [
+    setuptools
+  ];
 
   dependencies = [
     jmespath