Add CLI for uploading/downloading assets

Fix issue with (auth=True) in _get_session()

Abstract API call formatter

Remove duplicate code

Add support for include on get_asset_children()

Convert build_asset_info to private method

Add missing kwargs for pagination

(chore): Add pull request template
Add CLI

Add fiocli <- command line uploader/downloader

Author: jhodges10

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,13 @@
+## [DEVREL-XXXX]
+
+### Description:
+Please provide a short description of what this PR does
+
+### Depends on:
+- Does this PR depend on any other ones?
+
+### Includes changes from:
+- Does this PR includ changes from another PR?
+
+### I'd like feedback on:
+- What would you like feedback on?

README.md

@@ -24,6 +24,13 @@ $ git clone https://github.com/frameio/python-frameio-client
 $ pip install .
 ```
 
+### Developing
+Install the package into your development environment and link to it by running the following:
+
+```sh
+pipenv install -e . -pre
+```
+
 ## Documentation
 
 [Frame.io API Documentation](https://developer.frame.io/docs)
@@ -34,6 +41,27 @@ _Note: A valid token is required to make requests to Frame.io. Go to our [Develo
 
 In addition to the snippets below, examples are included in [/examples](/examples).
 
+### Use CLI
+When you install this package, a cli tool called `fioctl` will also be installed to your environment.
+
+**To upload a file or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--target "YOUR LOCAL SYSTEM DIRECTORY" \
+--threads 8
+```
+
+**To download a file, project, or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR LOCAL SYSTEM DIRECTORY" \
+--target "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--threads 2
+```
+
 ### Get User Info
 
 Get basic info on the authenticated user.

examples/recursive_upload.py

@@ -0,0 +1,94 @@
+import os
+import time
+import mimetypes
+import concurrent.futures
+import threading
+from frameioclient import FrameioClient
+from pprint import pprint
+
+global file_num
+file_num = 0
+
+global file_count
+file_count = 0
+
+def create_n_upload(task):
+    client=task[0]
+    file_p=task[1]
+    parent_asset_id=task[2]
+    abs_path = os.path.abspath(file_p)
+    file_s = os.path.getsize(file_p)
+    file_n = os.path.split(file_p)[1]
+    file_mime = mimetypes.guess_type(abs_path)[0]
+    
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=file_n,
+      type="file",
+      filetype=file_mime,
+      filesize=file_s
+    )
+    
+    with open(abs_path, "rb") as ul_file:
+        asset_info = client.upload(asset, ul_file)
+    
+    return asset_info
+
+
+def create_folder(folder_n, parent_asset_id):
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=folder_n,
+      type="folder",
+    )
+    
+    return asset['id']
+
+
+def file_counter(root_folder):
+    matches = []
+    for root, dirnames, filenames in os.walk(root_folder):
+        for filename in filenames:
+            matches.append(os.path.join(filename))
+
+    return matches
+
+
+def recursive_upload(client, folder, parent_asset_id):
+    # Seperate files and folders:
+    file_list = list()
+    folder_list = list()
+
+    for item in os.listdir(folder):
+        if item == ".DS_Store": # Ignore .DS_Store files on Mac
+            continue
+
+        complete_item_path = os.path.join(folder, item)
+
+        if os.path.isfile(complete_item_path):
+            file_list.append(item)
+        else:
+            folder_list.append(item)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        for file_p in file_list:
+            global file_num
+            file_num += 1
+            print(f"Starting {file_num}/{file_count}")
+            complete_dir_obj = os.path.join(folder, file_p)
+            task = (client, complete_dir_obj, parent_asset_id)
+            executor.submit(create_n_upload, task)
+
+    for folder_i in folder_list:
+        new_folder = os.path.join(folder, folder_i)
+        new_parent_asset_id = create_folder(folder_i, parent_asset_id)
+        recursive_upload(client, new_folder, new_parent_asset_id)
+
+
+if __name__ == "__main__":
+    root_folder = "./test_structure"
+    parent_asset_id = "PARENT_ASSET_ID"
+    client = FrameioClient(os.getenv("FRAME_IO_TOKEN"))
+
+    file_count = len(file_counter(root_folder))
+    recursive_upload(client, root_folder, parent_asset_id)

examples/upload_asset.py

@@ -1,11 +1,13 @@
+import os
 from frameioclient import FrameioClient
 
 def main():
-  client = FrameioClient("TOKEN")
-  file_path = './file_to_upload.mov'
-  parent_asset_id = ''
+  client = FrameioClient(os.getenv('FRAMEIO_TOKEN'))
+  file_path = '/Users/jeff/downloads/📼 Video/🎨 Graded Dailies/A001C034_210518_ROGH.mov'
+  parent_asset_id = 'b91433a2-8c9d-4da4-a9dd-80649cb91adf'
 
-  asset = client.assets.upload(parent_asset_id,file_path)
+  # asset = client.assets.upload(parent_asset_id,file_path)
+  client.assets.upload(parent_asset_id, file_path)
 
 if __name__ == "__main__":
   main()

frameioclient/__init__.py

@@ -1,9 +1,3 @@
 from .lib import *
-from .services import *
-from .client import FrameioClient
-<<<<<<<
-
-=======
 from .service import *
-from .lib import *
->>>>>>>
+from .client import FrameioClient

frameioclient/client.py

@@ -1,16 +1,14 @@
 from .lib import (
-  Utils,
   APIClient,
-  AWSClient,
   Telemetry,
   ClientVersion,
   ClientVersion,
   FrameioDownloader
 )
 
 class FrameioClient(APIClient, object):
-  def __init__(self, token, host):
-    super().__init__(token, host)
+  def __init__(self, token, host='https://api.frame.io', threads=5, progress=False):
+    super().__init__(token, host, threads, progress)
 
   @property
   def me(self):
@@ -34,41 +32,40 @@ def _download(self):
 
   @property
   def users(self):
-    from .services import User
+    from .service import User
     return User(self)
 
   @property
   def assets(self):
-    from .services import Asset
+    from .service import Asset
     return Asset(self)
-        total=3,
 
   @property
   def comments(self):
-    from .services import Comment
+    from .service import Comment
     return Comment(self)
 
   @property
   def logs(self):
-    from .services import AuditLogs
+    from .service import AuditLogs
     return AuditLogs(self)
 
   @property
   def review_links(self):
-    from .services import ReviewLink
+    from .service import ReviewLink
     return ReviewLink(self)
 
   @property
   def presentation_links(self):
-    from .services import PresentationLink
+    from .service import PresentationLink
     return PresentationLink(self)
 
   @property
   def projects(self):
-    from .services import Project
+    from .service import Project
     return Project(self)
 
   @property
   def teams(self):
-    from .services import Team
+    from .service import Team
     return Team(self)

frameioclient/fiocli.py

@@ -0,0 +1,52 @@
+import os
+import sys
+import argparse
+
+from frameioclient import FrameioClient
+
+
+def main():
+    parser=argparse.ArgumentParser(prog='fiocli', description='Frame.io Python SDK CLI')
+
+    ## Define args
+    parser.add_argument('--token', action='store', metavar='token', type=str, nargs='+', help='Developer Token')
+    # parser.add_argument('--op', action='store', metavar='op', type=str, nargs='+', help='Operation: upload, download')
+    parser.add_argument('--target', action='store', metavar='target', type=str, nargs='+', help='Target: remote project or folder, or alternatively a local file/folder')
+    parser.add_argument('--destination', action='store', metavar='destination', type=str, nargs='+', help='Destination: remote project or folder, or alternatively a local file/folder')
+    parser.add_argument('--threads', action='store', metavar='threads', type=int, nargs='+', help='Number of threads to use')
+
+    ## Parse args
+    args = parser.parse_args()
+
+    if args.threads:
+        threads = args.threads[0]
+    else:
+        threads = 5
+
+    ## Handle args
+    if args.token:
+        client = None
+        # print(args.token)
+        try:
+            client = FrameioClient(args.token[0], progress=True, threads=threads)
+        except Exception as e:
+            print("Failed")
+            sys.exit(1)
+
+        # If args.op == 'upload':
+        if args.target:
+            if args.destination:
+                # Check to see if this is a local target and thus a download
+                if os.path.isdir(args.destination[0]):
+                    asset = client.assets.get(args.target[0])
+                    return client.assets.download(asset, args.destination[0], progress=True, multi_part=True, concurrency=threads)
+                else: # This is an upload
+                    if os.path.isdir(args.target[0]):
+                        return client.assets.upload_folder(args.target[0], args.destination[0])
+                    else:
+                        return client.assets.upload(args.destination[0], args.target[0])
+            else:
+                print("No destination supplied")
+        else:
+            print("No target supplied")
+

frameioclient/lib/download.py

@@ -42,7 +42,7 @@ def __init__(self, asset, download_folder, prefix, multi_part=False, concurrency
     self.in_progress = 0
     self.filename = Utils.normalize_filename(asset["name"])
     self.request_logs = list()
-    self.session = AWSClient()._get_session(auth=None)
+    self.session = AWSClient()._get_session()
 
     self._evaluate_asset()
 
@@ -135,7 +135,7 @@ def single_part_download(self, url):
     print("Beginning download -- {} -- {}".format(self.asset["name"], Utils.format_bytes(self.file_size, type="size")))
 
     # Downloading
-    r = requests.get(url)
+    r = self.session.get(url)
     open(self.destination, "wb").write(r.content)
 
     download_time = time.time() - start_time