Working recursive project tree + download

Author: jhodges10

examples/download_project.py

@@ -0,0 +1,47 @@
+from frameioclient.lib.utils import Utils
+import os
+from pathlib import Path
+
+import pdb
+from time import time,sleep
+from pprint import pprint
+from frameioclient import FrameioClient
+
+def get_folder_size(path='.'):
+    total = 0
+    for entry in os.scandir(path):
+        if entry.is_file():
+            total += entry.stat().st_size
+        elif entry.is_dir():
+            total += get_folder_size(entry.path)
+    return total
+
+def demo_project_download(project_id):
+	TOKEN = os.getenv("FRAMEIO_TOKEN")
+	client = FrameioClient(TOKEN)
+
+	start_time = time()
+	download_dir = '/Volumes/Jeff-EXT/Python Transfer Test'
+	item_count = client.projects.download(project_id, destination_directory=download_dir)
+
+	# item_count = client.projects.download(project_id, destination_directory='/Users/jeff/Temp/Transfer vs Python SDK/Python SDK')
+
+	end_time = time()
+	elapsed = round((end_time - start_time), 2)
+
+	
+	folder_size = get_folder_size(download_dir)
+	# pdb.set_trace()
+
+	print(f"Found {item_count} items")
+	print(f"Took {elapsed} second to download {Utils.format_bytes(folder_size, type='size')} for project: {client.projects.get(project_id)['name']}")
+	print("\n")
+
+if __name__ == "__main__":
+	# project_id = '2dfb6ce6-90d8-4994-881f-f02cd94b1c81'
+	# project_id='e2845993-7330-54c6-8b77-eafbd5144eac'
+	# project_id = '5d3ff176-ab1f-4c0b-a027-abe3d2a960e3'
+	project_id = 'ba1791e8-bf1e-46cb-bcad-5e4bb6431a08'
+	demo_project_download(project_id)
+
+# Took 443.84 second to download 12.43 GB to USB HDD for project: HersheyPark Summer Campaign using Python SDK

examples/asset_tree.py renamed to examples/project_tree.py

@@ -10,7 +10,7 @@ def demo_folder_tree(project_id, slim):
 	client = FrameioClient(TOKEN)
 
 	start_time = time()
-	tree = client.projects.tree(project_id, slim)
+	tree = client.helpers.build_project_tree(project_id, slim=True)
 
 	end_time = time()
 	elapsed = round((end_time - start_time), 2)
@@ -24,8 +24,10 @@ def demo_folder_tree(project_id, slim):
 	print("\n")
 
 if __name__ == "__main__":
-	project_id = '2dfb6ce6-90d8-4994-881f-f02cd94b1c81'
+	# project_id = '2dfb6ce6-90d8-4994-881f-f02cd94b1c81'
 	# project_id='e2845993-7330-54c6-8b77-eafbd5144eac'
+	# project_id = '5d3ff176-ab1f-4c0b-a027-abe3d2a960e3'
+	project_id = 'ba1791e8-bf1e-46cb-bcad-5e4bb6431a08'
 	demo_folder_tree(project_id, slim=True)
 	# demo_folder_tree(project_id, slim=False)
 

frameioclient/service/assets.py

@@ -48,12 +48,14 @@ def get_children(self, asset_id, include=[], slim=False, **kwargs):
     """
     endpoint = '/assets/{}/children'.format(asset_id)
 
-
     if slim == True:
       query_params = ''
 
-      # Include children
-      query_params += '?' + 'include=children'
+      if len(include) > 0:
+        query_params += '?include={}'.format(include.join(','))
+      else:
+        # Include children
+        query_params += '?' + 'include=children'
 
       # Only fields
       query_params += '&' + 'only_fields=' + ','.join(constants.asset_excludes['only_fields'])
@@ -73,11 +75,7 @@ def get_children(self, asset_id, include=[], slim=False, **kwargs):
       endpoint += query_params
 
       # print("Final URL", endpoint)
-
-
-    if len(include) > 0:
-      endpoint += '?include={}'.format(include.join(','))
-
+      
     return self.client._api_call('get', endpoint, kwargs)
 
   def create(self, parent_asset_id, **kwargs):

frameioclient/service/helpers.py

@@ -1,7 +1,14 @@
+import os
+
+from pathlib import Path
+from time import time, sleep
+
 from .service import Service
-from typing import List
+from ..lib.utils import Utils
 
 from copy import deepcopy
+from typing import List
+from pprint import pprint
 
 class FrameioHelpers(Service):
 	def get_updated_assets(self, account_id, project_id, timestamp):
@@ -34,81 +41,110 @@ def get_updated_assets(self, account_id, project_id, timestamp):
 		endpoint = '/search/library'
 		return self.client._api_call('post', endpoint, payload=payload)
 
-	def get_assets_recursively(self, asset_id, slim, state: List):
+	def get_assets_recursively(self, asset_id, slim=True):
 		assets = self.client.assets.get_children(asset_id, slim)
-
-		print("Number of folders at top level", len(assets))
+		print("Number of assets at top level", len(assets))
 
 		for asset in assets:
-			print(asset['_type'], asset['name'])
+			print(f"Type: {asset['_type']}, Name: {asset['name']}, Children: {len(asset['children'])}")
+			total_bytes = 0
 
-			# We only get the first three items when we use "include=children" \
-			#  which is rather misleading
-			if asset['_type'] == "folder":
-				try:
-					if len(asset['children']) != asset['item_count']:
-						# Recursively fetch the contents of the folder
-						# No need to return the state here since it's shared memory
-						self.recursive_asset_fetch(asset['id'], slim, state)
-					else:
-						# Copy the asset before we clear out the children
-						asset_copy = deepcopy(asset)
-						asset_copy.pop('children')
-						state.append(asset_copy)
-				except KeyError as e:
-					# print(e)
-					print(asset.keys())
-
-			try:
-				children = asset['children']
-				for asset in children:
-					# if asset['_type'] == "folder":
-					# 	# Recursively fetch the contents of the folder
-					# 	# No need to return the state here since it's shared memory
-					# 	self.recursive_asset_fetch(asset['id'], slim, state)
-
-					if asset['_type'] == "file":
-						# Append the non-folder asset
-						state.append(asset)
-
-					if asset['_type'] == "verson_stack":
-						print("Grabbing top item from version stack")
-						versions = self.client.assets.get_children(asset['id'])
-						state.append(versions[0])
-
-			except KeyError as e:
-				print(e)
-				# Children not found for this asset, continuing
-				pass
+			if asset['_type'] == "file":
+				# Don't do nothing, it's a file!
+				continue
 
-			# if asset['_type'] == "folder":
-			# 	# Recursively fetch the contents of the folder
-			# 	# No need to return the state here since it's shared memory
-			# 	self.recursive_asset_fetch(asset['id'], slim, state)
+			if asset['_type'] == "verson_stack":
+				print("Grabbing top item from version stack")
+				versions = self.client.assets.get_children(asset['id'])
+				asset = versions[0] # re-assign on purpose
+				continue
 
-			# if asset['_type'] == "file":
-			# 	# Append the non-folder asset
-			# 	state.append(asset)
-
-			# if asset['_type'] == "verson_stack":
-			# 	print("Grabbing top item from version stack")
-			# 	versions = self.client.assets.get_children(asset['id'])
-			# 	state.append(versions[0])
-
-		return state
+			# We only get the first three items when we use "include=children"
+			if asset['_type'] == "folder":
+				# try:
+				if asset['item_count'] > 3:
+					# Recursively fetch the contents of the folder because we have to
+					# asset = deepcopy(asset)
+					# asset.pop('children')
+					asset['children'] = self.get_assets_recursively(asset['id'], slim)
+					# asset.update({'children': self.get_assets_recursively(asset['id'], slim)})
+					print("Grabbed more items for this sub dir")
+
+				else:
+					for i in asset['children']:
+						# If a folder is found, we still need to recursively search it
+						if i['_type'] == "folder":
+							i['children'] = self.get_assets_recursively(i['id'], slim)
+
+				# except KeyError as e:
+				# 	# No children found in this folder, move on
+				# 	print(e)
+				# 	continue
+
+		return assets
 
 	def build_project_tree(self, project_id, slim=False):
 		# if slim == True:
 		# 	self.client.assets.get_children()
 
 		# Get project info
 		project = self.client.projects.get(project_id)
-		root_asset_id = project['root_asset_id']
 
 		# Get children
-		initial_tree = self.get_assets_recursively(root_asset_id, slim, state=[])
+		initial_tree = self.get_assets_recursively(project['root_asset_id'], slim)
 
 		return initial_tree
 
+	def download_project(self, project_id, destination):
+		project = self.client.projects.get(project_id)
+		initial_tree = self.get_assets_recursively(project['root_asset_id'], slim=True)
+		self.recursive_downloader(destination, initial_tree)
+		# pprint(initial_tree)
+		# print(f"Downloading {Utils.format_bytes(total_bytes, type='size')}")
+
+	def recursive_downloader(self, directory, asset, count=0):
+		# TODO resolve this clusterfuck of downloads
+		print(f"Directory {directory}")
+
+		try:
+			# First check to see if we need to make the directory
+			target_directory = os.path.join(os.path.curdir, directory)
+			if not os.path.isdir(target_directory):
+				os.mkdir(os.path.abspath(target_directory))
+
+		except Exception as e:
+			target_directory = os.path.abspath(os.path.join(os.path.curdir, directory))
+			print(e)
+
+		if type(asset) == list:
+			for i in asset:
+				self.recursive_downloader(directory, i)
+
+		else:
+			try:
+				if asset['_type'] == 'folder':
+					if len(asset['children']) >= 0:
+						count += 1
+						# Create the new folder that these items will go in before it's too late
+						if not os.path.exists(os.path.join(target_directory, asset['name'])):
+							print("Path doesn't exist")
+							new_path = Path(target_directory, str(asset['name']).replace('/', '-'))
+							print(new_path.absolute)
+							print("Making new directory")
+							Path.mkdir(new_path)
+							sleep(2)
+
+						# Pass along the new directory they'll be living in and the children
+						self.recursive_downloader(f"{directory}/{str(asset['name']).replace('/', '-')}", asset['children'])
+
+				if asset['_type'] == 'file':
+					count += 1
+					return self.client.assets.download(asset, target_directory, multi_part=True, concurrency=10)
+
+			except Exception as e:
+				print(e)
+
+		return True
+
 if __name__ == "__main__":
 	pass

frameioclient/service/projects.py

@@ -61,6 +61,21 @@ def tree(self, project_id, slim):
 
     return FrameioHelpers(self.client).build_project_tree(project_id, slim)
 
+  def download(self, project_id, destination_directory='downloads'):
+    """
+    Download the provided project to disk.
+
+    :Args:
+      project_id (uuid): The project's id.
+      destination_directory (string): Directory on disk that you want to download the project to.
+
+      Example::
+        client.projects.download(
+          project_id="123",
+          destination_directory="./downloads"
+        )
+    """
+    return FrameioHelpers(self.client).download_project(project_id, destination=destination_directory)
 
   def get_collaborators(self, project_id, **kwargs):
     """