Parity with v2.0.0-alpha.19

.travis.yml

@@ -15,19 +15,18 @@ env:
   global:
     - PATH=$HOME/cached-deps:$PATH
   jobs:
-    - PACHYDERM_VERSION=2.0.0-alpha.12
-
+    - PACHYDERM_VERSION=$(jq -r .pachyderm version.json)
 jobs:
   include:
     # other tests
     - python: 3.8
       env:
         - TOXENV=lint
-        - PACHYDERM_VERSION=2.0.0-alpha.12
+        - PACHYDERM_VERSION=$(jq -r .pachyderm version.json)
     - python: 3.8
       env:
         - TOXENV=examples
-        - PACHYDERM_VERSION=2.0.0-alpha.12
+        - PACHYDERM_VERSION=$(jq -r .pachyderm version.json)
 install:
 - make ci-install
 before_script:

etc/proto_lint/proto_lint.py

@@ -63,6 +63,15 @@
 
 # A list of methods that we do not expect the library to implement
 BLACKLISTED_METHODS = {
+    Service.AUTH: [
+        # TODO: ignoring to pass proto_lint, but come back later
+        "delete_expired_auth_tokens",
+        "get_permissions",
+        "get_permissions_for_principal",
+        "revoke_auth_tokens_for_user",
+        "rotate_root_token",
+        "get_groups_for_principal",
+    ],
     Service.PFS: [
         # delete_all is ignored because we implement PPS' delete_all anyway
         "delete_all",
@@ -71,20 +80,24 @@
         "put_tar",
         "get_tar",
         "activate_auth",
+        "run_load_test",
+        # GetFileTar replaced GetFile, but GetFile will come back
+        "get_file_t_a_r",
+        "get_file",
         # TODO: add these new API methods
-        "renew_fileset",
-        "squash_commit",
-        "clear_commit",
-        "modify_file",
         "add_fileset",
-        "get_fileset",
+        "clear_commit",
         "create_fileset",
+        "get_fileset",
+        "modify_file",
+        "renew_fileset",
+        "squash_commit",
     ],
     Service.PPS: [
         # the following are ignored because they're for internal use only
         "activate_auth",
-        "create_job",
-        "update_job_state",
+        "create_pipeline_job",
+        "update_pipeline_job_state",
     ],
     Service.ENTERPRISE: [
         # internal RPC only
@@ -271,8 +284,8 @@
         ("pod_spec", None),
         ("transform", None),
     ],
-    "delete_job": [
-        ("job", "job_id"),
+    "delete_pipeline_job": [
+        ("pipeline_job", "job_id"),
     ],
     "delete_pipeline": [
         ("pipeline", "pipeline_name"),
@@ -281,51 +294,49 @@
     "delete_secret": [
         ("secret", "secret_name"),
     ],
-    "flush_job": [
+    "flush_pipeline_job": [
         ("to_pipelines", "pipeline_names"),
     ],
     "get_job_logs": [
-        ("job", "job_id"),
+        ("pipeline_job", "job_id"),
         ("master", None),
         ("pipeline", None),
     ],
     "get_pipeline_logs": [
         ("pipeline", ("pipeline_name")),
-        ("job", None),
+        ("pipeline_job", None),
     ],
     "inspect_datum": [
         ("datum", ("job_id", "datum_id")),
     ],
-    "inspect_job": [
-        ("job", "job_id"),
+    "inspect_pipeline_job": [
+        ("pipeline_job", "job_id"),
     ],
     "inspect_pipeline": [
         ("pipeline", "pipeline_name"),
         (None, "history"),
     ],
     "list_datum": [
-        ("job", "job_id"),
+        ("pipeline_job", "job_id"),
     ],
     "list_pipeline": [
         ("pipeline", None),
     ],
-    "list_job": [
+    "list_pipeline_job": [
         ("pipeline", "pipeline_name"),
     ],
     "restart_datum": [
-        ("job", "job_id"),
-    ],
-    "run_pipeline": [
-        ("pipeline", "pipeline_name"),
+        ("pipeline_job", "job_id"),
     ],
+    "run_pipeline": [("pipeline", "pipeline_name"), ("pipeline_job_id", "job_id")],
     "run_cron": [
         ("pipeline", "pipeline_name"),
     ],
     "start_pipeline": [
         ("pipeline", "pipeline_name"),
     ],
-    "stop_job": [
-        ("job", "job_id"),
+    "stop_pipeline_job": [
+        ("pipeline_job", "job_id"),
     ],
     "stop_pipeline": [
         ("pipeline", "pipeline_name"),

src/python_pachyderm/client.py

@@ -61,6 +61,7 @@ def __init__(
         root_certs=None,
         transaction_id=None,
         tls=None,
+        max_message_length=(1024 ** 2) * 20,
     ):
         """
         Creates a Pachyderm client.
@@ -104,6 +105,9 @@ def __init__(
             resp = self.authenticate_id_token(os.environ.get("PACH_PYTHON_OIDC_TOKEN"))
             self._auth_token = resp
             self._metadata = self._build_metadata()
+        self._grpc_options = [
+            ("grpc.max_receive_message_length", max_message_length),
+        ]
 
     @classmethod
     def new_in_cluster(cls, auth_token=None, transaction_id=None):
@@ -279,9 +283,16 @@ def _req(self, grpc_service, grpc_method_name, req=None, **kwargs):
             if self.root_certs:
                 ssl_channel_credentials = grpc_module.grpc.ssl_channel_credentials
                 ssl = ssl_channel_credentials(root_certificates=self.root_certs)
-                channel = grpc_module.grpc.secure_channel(self.address, ssl)
+                channel = grpc_module.grpc.secure_channel(
+                    self.address,
+                    ssl,
+                    options=self._grpc_options,
+                )
             else:
-                channel = grpc_module.grpc.insecure_channel(self.address)
+                channel = grpc_module.grpc.insecure_channel(
+                    self.address,
+                    options=self._grpc_options,
+                )
             stub = grpc_service.stub(channel)
             self._stubs[grpc_service] = stub
 

src/python_pachyderm/mixin/auth.py

@@ -165,7 +165,7 @@ def modify_members(self, group, add=None, remove=None):
             remove=remove or [],
         )
 
-    def get_groups(self, username=None):
+    def get_groups(self):
         """
         Gets which groups the given `username` belongs to. Returns a list of
         strings.
@@ -174,7 +174,7 @@ def get_groups(self, username=None):
 
         `username`: A string.
         """
-        return self._req(Service.AUTH, "GetGroups", username=username).groups
+        return self._req(Service.AUTH, "GetGroups").groups
 
     def get_users(self, group):
         """

src/python_pachyderm/mixin/license.py

@@ -22,7 +22,15 @@ def activate_license(self, activation_code, expires=None):
             expires=expires,
         )
 
-    def add_cluster(self, id, address, secret=None):
+    def add_cluster(
+        self,
+        id,
+        address,
+        secret=None,
+        user_address=None,
+        cluster_deployment_id=None,
+        enterprise_server=None,
+    ):
         """
         Register a cluster with the license service.
 
@@ -35,10 +43,19 @@ def add_cluster(self, id, address, secret=None):
         If not specified, a random secret will be generated and returned.
         """
         return self._req(
-            Service.LICENSE, "AddCluster", id=id, address=address, secret=secret
+            Service.LICENSE,
+            "AddCluster",
+            id=id,
+            address=address,
+            secret=secret,
+            user_address=user_address,
+            cluster_deployment_id=cluster_deployment_id,
+            enterprise_server=enterprise_server,
         )
 
-    def update_cluster(self, id, address):
+    def update_cluster(
+        self, id, address, user_address=None, cluster_deployment_id=None
+    ):
         """
         Update a cluster registered with the license service.
 
@@ -47,7 +64,14 @@ def update_cluster(self, id, address):
         * `id`: The unique ID to identify the cluster.
         * `address`: A GRPC address for the license server to reach the cluster.
         """
-        return self._req(Service.LICENSE, "UpdateCluster", id=id, address=address)
+        return self._req(
+            Service.LICENSE,
+            "UpdateCluster",
+            id=id,
+            address=address,
+            user_address=user_address,
+            cluster_deployment_id=cluster_deployment_id,
+        )
 
     def delete_cluster(self, id):
         """
@@ -76,3 +100,9 @@ def delete_all_license(self):
         Remove all clusters and deactivate the license service.
         """
         return self._req(Service.LICENSE, "DeleteAll")
+
+    def list_user_clusters(self):
+        """
+        Lists all clusters available to user.
+        """
+        return self._req(Service.LICENSE, "ListUserClusters")