Add fiocli <- command line uploader/downloader

Author: jhodges10

README.md

@@ -24,6 +24,13 @@ $ git clone https://github.com/frameio/python-frameio-client
 $ pip install .
 ```
 
+### Developing
+Install the package into your development environment and link to it by running the following:
+
+```sh
+pipenv install -e . -pre
+```
+
 ## Documentation
 
 [Frame.io API Documentation](https://developer.frame.io/docs)
@@ -34,6 +41,27 @@ _Note: A valid token is required to make requests to Frame.io. Go to our [Develo
 
 In addition to the snippets below, examples are included in [/examples](/examples).
 
+### Use CLI
+When you install this package, a cli tool called `fioctl` will also be installed to your environment.
+
+**To upload a file or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--target "YOUR LOCAL SYSTEM DIRECTORY" \
+--threads 8
+```
+
+**To download a file, project, or folder**
+```sh
+fioctl \
+--token fio-u-YOUR_TOKEN_HERE  \
+--destination "YOUR LOCAL SYSTEM DIRECTORY" \
+--target "YOUR TARGET FRAME.IO PROJECT OR FOLDER" \
+--threads 2
+```
+
 ### Get User Info
 
 Get basic info on the authenticated user.

examples/recursive_upload.py

@@ -0,0 +1,94 @@
+import os
+import time
+import mimetypes
+import concurrent.futures
+import threading
+from frameioclient import FrameioClient
+from pprint import pprint
+
+global file_num
+file_num = 0
+
+global file_count
+file_count = 0
+
+def create_n_upload(task):
+    client=task[0]
+    file_p=task[1]
+    parent_asset_id=task[2]
+    abs_path = os.path.abspath(file_p)
+    file_s = os.path.getsize(file_p)
+    file_n = os.path.split(file_p)[1]
+    file_mime = mimetypes.guess_type(abs_path)[0]
+    
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=file_n,
+      type="file",
+      filetype=file_mime,
+      filesize=file_s
+    )
+    
+    with open(abs_path, "rb") as ul_file:
+        asset_info = client.upload(asset, ul_file)
+    
+    return asset_info
+
+
+def create_folder(folder_n, parent_asset_id):
+    asset = client.create_asset(
+      parent_asset_id=parent_asset_id,  
+      name=folder_n,
+      type="folder",
+    )
+    
+    return asset['id']
+
+
+def file_counter(root_folder):
+    matches = []
+    for root, dirnames, filenames in os.walk(root_folder):
+        for filename in filenames:
+            matches.append(os.path.join(filename))
+
+    return matches
+
+
+def recursive_upload(client, folder, parent_asset_id):
+    # Seperate files and folders:
+    file_list = list()
+    folder_list = list()
+
+    for item in os.listdir(folder):
+        if item == ".DS_Store": # Ignore .DS_Store files on Mac
+            continue
+
+        complete_item_path = os.path.join(folder, item)
+
+        if os.path.isfile(complete_item_path):
+            file_list.append(item)
+        else:
+            folder_list.append(item)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        for file_p in file_list:
+            global file_num
+            file_num += 1
+            print(f"Starting {file_num}/{file_count}")
+            complete_dir_obj = os.path.join(folder, file_p)
+            task = (client, complete_dir_obj, parent_asset_id)
+            executor.submit(create_n_upload, task)
+
+    for folder_i in folder_list:
+        new_folder = os.path.join(folder, folder_i)
+        new_parent_asset_id = create_folder(folder_i, parent_asset_id)
+        recursive_upload(client, new_folder, new_parent_asset_id)
+
+
+if __name__ == "__main__":
+    root_folder = "./test_structure"
+    parent_asset_id = "PARENT_ASSET_ID"
+    client = FrameioClient(os.getenv("FRAME_IO_TOKEN"))
+
+    file_count = len(file_counter(root_folder))
+    recursive_upload(client, root_folder, parent_asset_id)

examples/upload_asset.py

@@ -1,11 +1,13 @@
+import os
 from frameioclient import FrameioClient
 
 def main():
-  client = FrameioClient("TOKEN")
-  file_path = './file_to_upload.mov'
-  parent_asset_id = ''
+  client = FrameioClient(os.getenv('FRAMEIO_TOKEN'))
+  file_path = '/Users/jeff/downloads/📼 Video/🎨 Graded Dailies/A001C034_210518_ROGH.mov'
+  parent_asset_id = 'b91433a2-8c9d-4da4-a9dd-80649cb91adf'
 
-  asset = client.assets.upload(parent_asset_id,file_path)
+  # asset = client.assets.upload(parent_asset_id,file_path)
+  client.assets.upload(parent_asset_id, file_path)
 
 if __name__ == "__main__":
   main()

frameioclient/__init__.py

@@ -1,3 +1,3 @@
 from .lib import *
-from .services import *
+from .service import *
 from .client import FrameioClient

frameioclient/client.py

@@ -7,8 +7,8 @@
 )
 
 class FrameioClient(APIClient, object):
-  def __init__(self, token, host='https://api.frame.io'):
-    super().__init__(token, host)
+  def __init__(self, token, host='https://api.frame.io', threads=5, progress=False):
+    super().__init__(token, host, threads, progress)
 
   @property
   def me(self):
@@ -32,40 +32,40 @@ def _download(self):
 
   @property
   def users(self):
-    from .services import User
+    from .service import User
     return User(self)
 
   @property
   def assets(self):
-    from .services import Asset
+    from .service import Asset
     return Asset(self)
 
   @property
   def comments(self):
-    from .services import Comment
+    from .service import Comment
     return Comment(self)
 
   @property
   def logs(self):
-    from .services import AuditLogs
+    from .service import AuditLogs
     return AuditLogs(self)
 
   @property
   def review_links(self):
-    from .services import ReviewLink
+    from .service import ReviewLink
     return ReviewLink(self)
 
   @property
   def presentation_links(self):
-    from .services import PresentationLink
+    from .service import PresentationLink
     return PresentationLink(self)
 
   @property
   def projects(self):
-    from .services import Project
+    from .service import Project
     return Project(self)
 
   @property
   def teams(self):
-    from .services import Team
+    from .service import Team
     return Team(self)

frameioclient/fioctl.py renamed to frameioclient/fiocli.py

@@ -6,34 +6,45 @@
 
 
 def main():
-    parser=argparse.ArgumentParser(prog='fioctl', description='Frame.io Python SDK CLI')
+    parser=argparse.ArgumentParser(prog='fiocli', description='Frame.io Python SDK CLI')
 
     ## Define args
     parser.add_argument('--token', action='store', metavar='token', type=str, nargs='+', help='Developer Token')
-    parser.add_argument('--op', action='store', metavar='op', type=str, nargs='+', help='Operation: upload, download')
+    # parser.add_argument('--op', action='store', metavar='op', type=str, nargs='+', help='Operation: upload, download')
     parser.add_argument('--target', action='store', metavar='target', type=str, nargs='+', help='Target: remote project or folder, or alternatively a local file/folder')
     parser.add_argument('--destination', action='store', metavar='destination', type=str, nargs='+', help='Destination: remote project or folder, or alternatively a local file/folder')
+    parser.add_argument('--threads', action='store', metavar='threads', type=int, nargs='+', help='Number of threads to use')
 
     ## Parse args
     args = parser.parse_args()
 
-    # ## Handle args
+    if args.threads:
+        threads = args.threads[0]
+    else:
+        threads = 5
+
+    ## Handle args
     if args.token:
         client = None
+        # print(args.token)
         try:
-            client = FrameioClient("fio-u-dVwbtNcxXCxySPRIRadPflefXpV-NgVD4vqR3EGIWg20xIkGd_6rQ7pDNFA5YOCy")
+            client = FrameioClient(args.token[0], progress=True, threads=threads)
         except Exception as e:
             print("Failed")
             sys.exit(1)
 
-        # if args.op == 'upload':
+        # If args.op == 'upload':
         if args.target:
             if args.destination:
                 # Check to see if this is a local target and thus a download
                 if os.path.isdir(args.destination[0]):
-                    return client.assets.download(args.target[0], args.destination[0])
+                    asset = client.assets.get(args.target[0])
+                    return client.assets.download(asset, args.destination[0], progress=True, multi_part=True, concurrency=threads)
                 else: # This is an upload
-                    return client.assets.upload(args.destination[0], args.target[0])
+                    if os.path.isdir(args.target[0]):
+                        return client.assets.upload_folder(args.target[0], args.destination[0])
+                    else:
+                        return client.assets.upload(args.destination[0], args.target[0])
             else:
                 print("No destination supplied")
         else:

frameioclient/lib/transport.py

@@ -44,10 +44,12 @@ def _get_session(self):
 
 
 class APIClient(HTTPClient, object):
-    def __init__(self, token, host):
+    def __init__(self, token, host, threads, progress):
         super().__init__()
         self.host = host
         self.token = token
+        self.threads = threads
+        self.progress = progress
         self._initialize_thread()
         self.session = self._get_session()
         self.auth_header = {
@@ -84,8 +86,8 @@ def _api_call(self, method, endpoint, payload={}, limit=None):
 
             return r.json()
 
-        if r.status_code == 422 and "presentation" in endpoint:
-            raise PresentationException
+        # if r.status_code == 422 and "presentation" in endpoint:
+        #     raise PresentationException
 
         return r.raise_for_status()
 

frameioclient/lib/upload.py

@@ -4,13 +4,17 @@
 import threading
 import concurrent.futures
 
+from .utils import Utils
+
 thread_local = threading.local()
 
 class FrameioUploader(object):
-  def __init__(self, asset, file):
+  def __init__(self, asset=None, file=None):
     self.asset = asset
     self.file = file
     self.chunk_size = None
+    self.file_count = 0
+    self.file_num = 0
 
   def _calculate_chunks(self, total_size, chunk_count):
     self.chunk_size = int(math.ceil(total_size / chunk_count))
@@ -76,3 +80,49 @@ def upload(self):
 
         task = (url, chunk_offset, i)
         executor.submit(self._upload_chunk, task)
+
+  def file_counter(self, folder):
+      matches = []
+      for root, dirnames, filenames in os.walk(folder):
+          for filename in filenames:
+              matches.append(os.path.join(filename))
+
+      self.file_count = len(matches)
+
+      return matches
+
+  def recursive_upload(self, client, folder, parent_asset_id):
+      # Seperate files and folders:
+      file_list = list()
+      folder_list = list()
+
+      if self.file_count == 0:
+        self.file_counter(folder)
+
+      for item in os.listdir(folder):
+          if item == ".DS_Store": # Ignore .DS_Store files on Mac
+              continue
+
+          complete_item_path = os.path.join(folder, item)
+
+          if os.path.isfile(complete_item_path):
+              file_list.append(item)
+          else:
+              folder_list.append(item)
+
+      for file_p in file_list:
+          self.file_num += 1
+
+          complete_dir_obj = os.path.join(folder, file_p)
+          print(f"Starting {self.file_num:02d}/{self.file_count}, Size: {Utils.format_bytes(os.path.getsize(complete_dir_obj), type='size')}, Name: {file_p}")
+          client.assets.upload(parent_asset_id, complete_dir_obj)
+
+      for folder_name in folder_list:
+          new_folder = os.path.join(folder, folder_name)
+          new_parent_asset_id = client.assets.create(
+            parent_asset_id=parent_asset_id,
+            name=folder_name,
+            type="folder"
+          )['id']
+
+          self.recursive_upload(client, new_folder, new_parent_asset_id)